Update plugin management approach

devtools/miniconda_install.sh

@@ -22,7 +22,7 @@ conda_version="latest"
 #conda_version="4.4.10"  # can pin a miniconda version like this, if needed
 
 MINICONDA=Miniconda${pyV}-${conda_version}-${OS_ARCH}.sh
-MINICONDA_MD5=$(curl -s https://repo.continuum.io/miniconda/ | grep -A3 $MINICONDA | sed -n '4p' | sed -n 's/ *<td>\(.*\)<\/td> */\1/p')
+MINICONDA_MD5=$(curl -sL https://repo.continuum.io/miniconda/ | grep -A3 $MINICONDA | sed -n '4p' | sed -n 's/ *<td>\(.*\)<\/td> */\1/p')
 wget https://repo.continuum.io/miniconda/$MINICONDA
 SCRIPT_MD5=`eval "$MD5_CMD $MD5_OPT $MINICONDA" | cut -d ' ' -f 1`
 

paths_cli/cli.py

@@ -7,23 +7,15 @@
 import logging
 import logging.config
 import os
+import pathlib
 
 import click
 # import click_completion
 # click_completion.init()
 
-CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
-
-_POSSIBLE_PLUGIN_FOLDERS = [
-    os.path.join(os.path.dirname(__file__), 'commands'),
-    os.path.join(click.get_app_dir("OpenPathSampling"), 'cli-plugins'),
-    os.path.join(click.get_app_dir("OpenPathSampling", force_posix=True),
-                 'cli-plugins'),
-]
-
-OPSPlugin = collections.namedtuple("OPSPlugin",
-                                   ['name', 'filename', 'func', 'section'])
+from .plugin_management import FilePluginLoader, NamespacePluginLoader
 
+CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
 
 class OpenPathSamplingCLI(click.MultiCommand):
     """Main class for the command line interface
@@ -32,13 +24,20 @@ class OpenPathSamplingCLI(click.MultiCommand):
     """
     def __init__(self, *args, **kwargs):
         # the logic here is all about loading the plugins
-        self.plugin_folders = []
-        for folder in _POSSIBLE_PLUGIN_FOLDERS:
-            if folder not in self.plugin_folders and os.path.exists(folder):
-                self.plugin_folders.append(folder)
+        commands = str(pathlib.Path(__file__).parent.resolve() / 'commands')
+        def app_dir_plugins(posix):
+            return str(pathlib.Path(
+                click.get_app_dir("OpenPathSampling", force_posix=posix)
+            ).resolve() / 'cli-plugins')
+
+        self.plugin_loaders = [
+            FilePluginLoader(commands),
+            FilePluginLoader(app_dir_plugins(posix=False)),
+            FilePluginLoader(app_dir_plugins(posix=True)),
+            NamespacePluginLoader('paths_cli.plugins')
+        ]
 
-        plugin_files = self._list_plugin_files(self.plugin_folders)
-        plugins = self._load_plugin_files(plugin_files)
+        plugins = sum([loader() for loader in self.plugin_loaders], [])
 
         self._get_command = {}
         self._sections = collections.defaultdict(list)
@@ -62,45 +61,6 @@ def _deregister_plugin(self, plugin):
     def plugin_for_command(self, command_name):
         return {p.name: p for p in self.plugins}[command_name]
 
-    @staticmethod
-    def _list_plugin_files(plugin_folders):
-        def is_plugin(filename):
-            return (
-                filename.endswith(".py") and not filename.startswith("_")
-                and not filename.startswith(".")
-            )
-
-        plugin_files = []
-        for folder in plugin_folders:
-            files = [os.path.join(folder, f) for f in os.listdir(folder)
-                     if is_plugin(f)]
-            plugin_files += files
-        return plugin_files
-
-    @staticmethod
-    def _filename_to_command_name(filename):
-        command_name = filename[:-3]  # get rid of .py
-        command_name = command_name.replace('_', '-')  # commands use -
-        return command_name
-
-    @staticmethod
-    def _load_plugin(name):
-        ns = {}
-        with open(name) as f:
-            code = compile(f.read(), name, 'exec')
-            eval(code, ns, ns)
-        return ns['CLI'], ns['SECTION']
-
-    def _load_plugin_files(self, plugin_files):
-        plugins = []
-        for full_name in plugin_files:
-            _, filename = os.path.split(full_name)
-            command_name = self._filename_to_command_name(filename)
-            func, section = self._load_plugin(full_name)
-            plugins.append(OPSPlugin(name=command_name, filename=full_name,
-                                     func=func, section=section))
-        return plugins
-
     def list_commands(self, ctx):
         return list(self._get_command.keys())
 

paths_cli/plugin_management.py

@@ -0,0 +1,140 @@
+import collections
+import pkgutil
+import importlib
+import os
+
+OPSPlugin = collections.namedtuple(
+    "OPSPlugin", ['name', 'location', 'func', 'section', 'plugin_type']
+)
+
+class CLIPluginLoader(object):
+    """Abstract object for CLI plugins
+
+    The overall approach involves 5 steps, each of which can be overridden:
+
+    1. Find candidate plugins (which must be Python modules)
+    2. Load the namespaces associated into a dict (nsdict)
+    3. Based on those namespaces, validate that the module *is* a plugin
+    4. Get the associated command name
+    5. Return an OPSPlugin object for each plugin
+
+    Details on steps 1, 2, and 4 differ based on whether this is a
+    filesystem-based plugin or a namespace-based plugin.
+    """
+    def __init__(self, plugin_type, search_path):
+        self.plugin_type = plugin_type
+        self.search_path = search_path
+
+    def _find_candidates(self):
+        raise NotImplementedError()
+
+    @staticmethod
+    def _make_nsdict(candidate):
+        raise NotImplementedError()
+
+    @staticmethod
+    def _validate(nsdict):
+        for attr in ['CLI', 'SECTION']:
+            if attr not in nsdict:
+                return False
+        return True
+
+    def _get_command_name(self, candidate):
+        raise NotImplementedError()
+
+    def _find_valid(self):
+        candidates = self._find_candidates()
+        namespaces = {cand: self._make_nsdict(cand) for cand in candidates}
+        valid = {cand: ns for cand, ns in namespaces.items()
+                 if self._validate(ns)}
+        return valid
+
+    def __call__(self):
+        valid = self._find_valid()
+        plugins = [
+            OPSPlugin(name=self._get_command_name(cand),
+                      location=cand,
+                      func=ns['CLI'],
+                      section=ns['SECTION'],
+                      plugin_type=self.plugin_type)
+            for cand, ns in valid.items()
+        ]
+        return plugins
+
+
+class FilePluginLoader(CLIPluginLoader):
+    """File-based plugins (quick and dirty)
+
+    Parameters
+    ----------
+    search_path : str
+        path to the directory that contains plugins (OS-dependent format)
+    """
+    def __init__(self, search_path):
+        super().__init__(plugin_type="file", search_path=search_path)
+
+    def _find_candidates(self):
+        def is_plugin(filename):
+            return (
+                filename.endswith(".py") and not filename.startswith("_")
+                and not filename.startswith(".")
+            )
+
+        if not os.path.exists(os.path.join(self.search_path)):
+            return []
+
+        candidates = [os.path.join(self.search_path, f)
+                      for f in os.listdir(self.search_path)
+                      if is_plugin(f)]
+        return candidates
+
+    @staticmethod
+    def _make_nsdict(candidate):
+        ns = {}
+        with open(candidate) as f:
+            code = compile(f.read(), candidate, 'exec')
+            eval(code, ns, ns)
+        return ns
+
+    def _get_command_name(self, candidate):
+        _, command_name = os.path.split(candidate)
+        command_name = command_name[:-3]  # get rid of .py
+        command_name = command_name.replace('_', '-')  # commands use -
+        return command_name
+
+
+class NamespacePluginLoader(CLIPluginLoader):
+    """Load namespace plugins (plugins for wide distribution)
+
+    Parameters
+    ----------
+    search_path : str
+        namespace (dot-separated) where plugins can be found
+    """
+    def __init__(self, search_path):
+        super().__init__(plugin_type="namespace", search_path=search_path)
+
+    def _find_candidates(self):
+        # based on https://packaging.python.org/guides/creating-and-discovering-plugins/#using-namespace-packages
+        def iter_namespace(ns_pkg):
+            return pkgutil.iter_modules(ns_pkg.__path__,
+                                        ns_pkg.__name__ + ".")
+
+        ns = importlib.import_module(self.search_path)
+        candidates = [
+            importlib.import_module(name)
+            for _, name, _ in iter_namespace(ns)
+        ]
+        return candidates
+
+    @staticmethod
+    def _make_nsdict(candidate):
+        return vars(candidate)
+
+    def _get_command_name(self, candidate):
+        # +1 for the dot
+        command_name = candidate.__name__
+        command_name = command_name[len(self.search_path) + 1:]
+        command_name = command_name.replace('_', '-')  # commands use -
+        return command_name
+

paths_cli/tests/null_command.py

@@ -1,5 +1,8 @@
 import logging
 import click
+
+from paths_cli.plugin_management import FilePluginLoader, OPSPlugin
+
 @click.command(
     'null-command',
     short_help="Do nothing (testing)"
@@ -14,7 +17,13 @@ def null_command():
 class NullCommandContext(object):
     """Context that registers/deregisters the null command (for tests)"""
     def __init__(self, cli):
-        self.plugin = cli._load_plugin_files([__file__])[0]
+        self.plugin = OPSPlugin(name="null-command",
+                                location=__file__,
+                                func=CLI,
+                                section=SECTION,
+                                plugin_type="file")
+
+        cli._register_plugin(self.plugin)
         self.cli = cli
 
     def __enter__(self):

paths_cli/tests/test_cli.py

@@ -3,13 +3,11 @@
 from click.testing import CliRunner
 
 from paths_cli.cli import *
+from paths_cli.plugin_management import OPSPlugin
 from .null_command import NullCommandContext
 
 
 class TestOpenPathSamplingCLI(object):
-    # TODO: more thorough testing of private methods to find/register
-    # plugins might be nice; so far we mainly focus on testing the API.
-    # (Still have smoke tests covering everything, though.)
     def setup(self):
         def make_mock(name, helpless=False):
             mock = MagicMock(return_value=name)
@@ -21,22 +19,27 @@ def make_mock(name, helpless=False):
 
         self.plugin_dict = {
             'foo': OPSPlugin(name='foo',
-                             filename='foo.py',
+                             location='foo.py',
                              func=make_mock('foo'),
-                             section='Simulation'),
+                             section='Simulation',
+                             plugin_type='file'),
             'foo-bar': OPSPlugin(name='foo-bar',
-                                 filename='foo_bar.py',
+                                 location='foo_bar.py',
                                  func=make_mock('foobar', helpless=True),
-                                 section='Miscellaneous')
+                                 section='Miscellaneous',
+                                 plugin_type="file")
         }
-        self.fake_plugins = list(self.plugin_dict.values())
-        mock_plugins = MagicMock(return_value=self.fake_plugins)
-        with patch.object(OpenPathSamplingCLI, '_load_plugin_files',
-                          mock_plugins):
-            self.cli = OpenPathSamplingCLI()
+        self.plugins = list(self.plugin_dict.values())
+        self.cli = OpenPathSamplingCLI()
+        # need to copy the plugins since we're changing the list
+        for plugin in self.cli.plugins[:]:
+            self.cli._deregister_plugin(plugin)
+
+        for plugin in self.plugins:
+            self.cli._register_plugin(plugin)
 
     def test_plugins(self):
-        assert self.cli.plugins == self.fake_plugins
+        assert self.cli.plugins == self.plugins
         assert self.cli._sections['Simulation'] == ['foo']
         assert self.cli._sections['Miscellaneous'] == ['foo-bar']