Add support for installing plugins via VCS

fixes pybuilder#276, connected to pybuilder#276

src/main/python/pybuilder/core.py

@@ -207,11 +207,11 @@ def use_bldsup(build_support_dir="bldsup"):
     sys.path.insert(0, build_support_dir)
 
 
-def use_plugin(name, version=None):
+def use_plugin(name, version=None, plugin_module_name=None):
     from pybuilder.reactor import Reactor
     reactor = Reactor.current_instance()
     if reactor is not None:
-        reactor.require_plugin(name)
+        reactor.require_plugin(name, version, plugin_module_name)
 
 
 class Author(object):

src/main/python/pybuilder/errors.py

@@ -91,6 +91,12 @@ def __init__(self, plugin, message=""):
             "Missing plugin '%s': %s", plugin, message)
 
 
+class UnspecifiedPluginNameException(PyBuilderException):
+    def __init__(self, plugin):
+        super(UnspecifiedPluginNameException, self).__init__(
+            "Plugin module name is not specified '%s'", plugin)
+
+
 class IncompatiblePluginException(PyBuilderException):
     def __init__(self, plugin_name, required_pyb_version, actual_pyb_version):
         super(IncompatiblePluginException, self).__init__(

src/main/python/pybuilder/pluginloader.py

@@ -28,10 +28,15 @@
 from pip._vendor.packaging.version import Version
 
 from pybuilder import __version__ as pyb_version
-from pybuilder.errors import MissingPluginException, IncompatiblePluginException
+from pybuilder.errors import (MissingPluginException,
+                              IncompatiblePluginException,
+                              UnspecifiedPluginNameException,
+                              )
 from pybuilder.utils import execute_command, read_file
 
 PYPI_PLUGIN_PROTOCOL = "pypi:"
+VCS_PLUGIN_PROTOCOL = "vcs:"
+
 if pyb_version == "${dist_version}":  # This is the case of PyB bootstrap
     PYB_VERSION = Version('0.0.1.dev0')
 else:
@@ -42,26 +47,30 @@ class PluginLoader(object):
     def __init__(self, logger):
         self.logger = logger
 
-    def load_plugin(self, project, name, version=None):
+    def load_plugin(self, project, name, version=None, plugin_module_name=None):
         pass
 
 
 class BuiltinPluginLoader(PluginLoader):
-    def load_plugin(self, project, name, version=None):
+    def load_plugin(self, project, name, version=None, plugin_module_name=None):
         self.logger.debug("Trying to load builtin plugin '%s'", name)
-        builtin_plugin_name = "pybuilder.plugins.%s_plugin" % name
+        builtin_plugin_name = plugin_module_name or "pybuilder.plugins.%s_plugin" % name
 
         plugin_module = _load_plugin(builtin_plugin_name, name)
         self.logger.debug("Found builtin plugin '%s'", builtin_plugin_name)
         return plugin_module
 
 
 class ThirdPartyPluginLoader(PluginLoader):
-    def load_plugin(self, project, name, version=None):
+    def load_plugin(self, project, name, version=None, plugin_module_name=None):
         thirdparty_plugin = name
         # Maybe we already installed this plugin from PyPI before
         if thirdparty_plugin.startswith(PYPI_PLUGIN_PROTOCOL):
             thirdparty_plugin = thirdparty_plugin.replace(PYPI_PLUGIN_PROTOCOL, "")
+        elif thirdparty_plugin.startswith(VCS_PLUGIN_PROTOCOL):
+            if not plugin_module_name:
+                raise UnspecifiedPluginNameException(name)
+            thirdparty_plugin = plugin_module_name
         self.logger.debug("Trying to load third party plugin '%s'", thirdparty_plugin)
 
         plugin_module = _load_plugin(thirdparty_plugin, name)
@@ -70,52 +79,57 @@ def load_plugin(self, project, name, version=None):
 
 
 class DownloadingPluginLoader(ThirdPartyPluginLoader):
-    def load_plugin(self, project, name, version=None):
+    def load_plugin(self, project, name, version=None, plugin_module_name=None):
         display_name = "%s%s" % (name, " version %s" % version if version else "")
         self.logger.info("Downloading missing plugin {0}".format(display_name))
         try:
-            _install_external_plugin(name, version, self.logger)
+            _install_external_plugin(name, version, self.logger, plugin_module_name)
             self.logger.info("Installed plugin {0}.".format(display_name))
         except MissingPluginException as e:
             self.logger.error("Could not install plugin {0}: {1}.".format(display_name, e))
             return None
-        return ThirdPartyPluginLoader.load_plugin(self, project, name)
+        return ThirdPartyPluginLoader.load_plugin(self, project, name, version, plugin_module_name)
 
 
 class DispatchingPluginLoader(PluginLoader):
     def __init__(self, logger, *loader):
         super(DispatchingPluginLoader, self).__init__(logger)
         self.loader = loader
 
-    def load_plugin(self, project, name, version=None):
+    def load_plugin(self, project, name, version=None, plugin_module_name=None):
         last_problem = None
         for loader in self.loader:
             try:
-                return loader.load_plugin(project, name)
+                return loader.load_plugin(project, name, version, plugin_module_name)
             except MissingPluginException as e:
                 last_problem = e
         raise last_problem
 
 
-def _install_external_plugin(name, version, logger):
-    if not name.startswith(PYPI_PLUGIN_PROTOCOL):
+def _install_external_plugin(name, version, logger, plugin_module_name):
+    if not name.startswith(PYPI_PLUGIN_PROTOCOL) and not name.startswith(VCS_PLUGIN_PROTOCOL):
         message = "Only plugins starting with '{0}' are currently supported"
-        raise MissingPluginException(name, message.format(PYPI_PLUGIN_PROTOCOL))
-    plugin_name_on_pypi = name.replace(PYPI_PLUGIN_PROTOCOL, "")
-    log_file = tempfile.NamedTemporaryFile(delete=False).name
-    install_cmd = ['pip', 'install']
-    if version:
-        install_cmd += ['--upgrade', '--pre', plugin_name_on_pypi + str(version)]
-    else:
-        install_cmd += [plugin_name_on_pypi]
-    result = execute_command(install_cmd,
-                             log_file,
-                             error_file_name=log_file,
-                             shell=True)
-    if result != 0:
-        logger.error("The following pip error was encountered:\n" + "".join(read_file(log_file)))
-        message = "Failed to install from PyPI".format(plugin_name_on_pypi)
-        raise MissingPluginException(name, message)
+        raise MissingPluginException(name, message.format((PYPI_PLUGIN_PROTOCOL, VCS_PLUGIN_PROTOCOL)))
+
+    if name.startswith(PYPI_PLUGIN_PROTOCOL):
+        pip_package = name.replace(PYPI_PLUGIN_PROTOCOL, "")
+        if version:
+            pip_package += str(version)
+    elif name.startswith(VCS_PLUGIN_PROTOCOL):
+        pip_package = name.replace(VCS_PLUGIN_PROTOCOL, "")
+
+    with tempfile.NamedTemporaryFile(delete=False) as log_file:
+        log_file_name = log_file.name
+        install_cmd = ['pip', 'install', '--upgrade', pip_package]
+        result = execute_command(install_cmd,
+                                 log_file_name,
+                                 error_file_name=log_file_name,
+                                 cwd=".",
+                                 shell=False)
+        if result != 0:
+            logger.error("The following pip error was encountered:\n" + "".join(read_file(log_file_name)))
+            message = "Failed to install plugin from {0}".format(pip_package)
+            raise MissingPluginException(name, message)
 
 
 def _load_plugin(plugin_module_name, plugin_name):

src/main/python/pybuilder/reactor.py

@@ -68,11 +68,11 @@ def __init__(self, logger, execution_manager, plugin_loader=None):
         self._plugins = []
         self.project = None
 
-    def require_plugin(self, plugin, version=None):
+    def require_plugin(self, plugin, version=None, plugin_module_name=None):
         if plugin not in self._plugins:
             try:
                 self._plugins.append(plugin)
-                self.import_plugin(plugin, version)
+                self.import_plugin(plugin, version, plugin_module_name)
             except:  # NOQA
                 self._plugins.remove(plugin)
                 raise
@@ -199,9 +199,9 @@ def log_project_properties(self):
             formatted += "\n%40s : %s" % (key, self.project.get_property(key))
         self.logger.debug("Project properties: %s", formatted)
 
-    def import_plugin(self, plugin, version=None):
+    def import_plugin(self, plugin, version=None, plugin_module_name=None):
         self.logger.debug("Loading plugin '%s'%s", plugin, " version %s" % version if version else "")
-        plugin_module = self.plugin_loader.load_plugin(self.project, plugin, version)
+        plugin_module = self.plugin_loader.load_plugin(self.project, plugin, version, plugin_module_name)
         self.collect_tasks_and_actions_and_initializers(plugin_module)
 
     def collect_tasks_and_actions_and_initializers(self, project_module):

src/unittest/python/pluginloader_tests.py

@@ -31,7 +31,7 @@
 from mockito import when, verify, unstub, never  # TODO @mriehl get rid of mockito here
 from mock import patch, Mock, ANY
 from test_utils import mock  # TODO @mriehl WTF is this sorcery?!
-from pybuilder.errors import MissingPluginException, IncompatiblePluginException
+from pybuilder.errors import MissingPluginException, IncompatiblePluginException, UnspecifiedPluginNameException
 from pybuilder.pluginloader import (BuiltinPluginLoader,
                                     DispatchingPluginLoader,
                                     ThirdPartyPluginLoader,
@@ -133,32 +133,60 @@ def test_should_download_module_from_pypi(self, install, _):
         logger = Mock()
         DownloadingPluginLoader(logger).load_plugin(Mock(), "pypi:external_plugin")
 
-        install.assert_called_with("pypi:external_plugin", None, logger)
+        install.assert_called_with("pypi:external_plugin", None, logger, None)
 
     @patch("pybuilder.pluginloader.ThirdPartyPluginLoader.load_plugin")
     @patch("pybuilder.pluginloader._install_external_plugin")
-    def test_should_load_module_after_downloading_when_download_succeeds(self, _, load):
+    def test_should_load_module_after_downloading_with_pypi_when_download_succeeds(self, _, load):
         project = Mock()
         downloader = DownloadingPluginLoader(Mock())
         plugin = downloader.load_plugin(project, "pypi:external_plugin")
 
-        load.assert_called_with(downloader, project, "pypi:external_plugin")
+        load.assert_called_with(downloader, project, "pypi:external_plugin", None, None)
         self.assertEquals(plugin, load.return_value)
 
     @patch("pybuilder.pluginloader.ThirdPartyPluginLoader.load_plugin")
     @patch("pybuilder.pluginloader._install_external_plugin")
-    def test_should_not_load_module_after_downloading_when_download_fails(self, install, load):
-        install.side_effect = MissingPluginException("BOOM")
+    def test_should_not_load_module_after_downloading_when_pypi_download_fails(self, install, load):
+        install.side_effect = MissingPluginException("PyPI BOOM")
         downloader = DownloadingPluginLoader(Mock())
         plugin = downloader.load_plugin(Mock(), "pypi:external_plugin")
 
         self.assertFalse(load.called)
         self.assertEquals(plugin, None)
 
+    @patch("pybuilder.pluginloader.ThirdPartyPluginLoader.load_plugin")
+    @patch("pybuilder.pluginloader._install_external_plugin")
+    def test_should_not_load_module_after_downloading_when_vcs_download_fails(self, install, load):
+        install.side_effect = MissingPluginException("VCS BOOM")
+        downloader = DownloadingPluginLoader(Mock())
+        plugin = downloader.load_plugin(Mock(), "vcs:external_plugin URL")
+
+        self.assertFalse(load.called)
+        self.assertEquals(plugin, None)
+
+    @patch("pybuilder.pluginloader._load_plugin")
+    @patch("pybuilder.pluginloader._install_external_plugin")
+    def test_should_fail_with_vcs_when_no_plugin_module_specified(self, _, load):
+        project = Mock()
+        downloader = DownloadingPluginLoader(Mock())
+
+        self.assertRaises(UnspecifiedPluginNameException, downloader.load_plugin, project, "vcs:external_plugin URL")
+
+    @patch("pybuilder.pluginloader._load_plugin")
+    @patch("pybuilder.pluginloader._install_external_plugin")
+    def test_should_load_module_after_downloading_with_vcs_when_download_succeeds(self, _, load):
+        project = Mock()
+        downloader = DownloadingPluginLoader(Mock())
+        plugin = downloader.load_plugin(project, "vcs:external_plugin URL", plugin_module_name="external_plugin_module")
+
+        load.assert_called_with("external_plugin_module", "vcs:external_plugin URL")
+        self.assertEquals(plugin, load.return_value)
+
 
 class InstallExternalPluginTests(unittest.TestCase):
     def test_should_raise_error_when_protocol_is_invalid(self):
-        self.assertRaises(MissingPluginException, _install_external_plugin, "some-plugin", None, Mock())
+        self.assertRaises(MissingPluginException, _install_external_plugin, "some-plugin", None, Mock(), None)
 
     @patch("pybuilder.pluginloader.read_file")
     @patch("pybuilder.pluginloader.tempfile")
@@ -167,9 +195,10 @@ def test_should_install_plugin(self, execute, _, read_file):
         read_file.return_value = ["no problems", "so far"]
         execute.return_value = 0
 
-        _install_external_plugin("pypi:some-plugin", None, Mock())
+        _install_external_plugin("pypi:some-plugin", None, Mock(), None)
 
-        execute.assert_called_with(['pip', 'install', 'some-plugin'], ANY, shell=True, error_file_name=ANY)
+        execute.assert_called_with(['pip', 'install', '--upgrade', 'some-plugin'], ANY, shell=False,
+                                   error_file_name=ANY, cwd=".")
 
     @patch("pybuilder.pluginloader.read_file")
     @patch("pybuilder.pluginloader.tempfile")
@@ -178,10 +207,34 @@ def test_should_install_plugin_with_version(self, execute, _, read_file):
         read_file.return_value = ["no problems", "so far"]
         execute.return_value = 0
 
-        _install_external_plugin("pypi:some-plugin", "===1.2.3", Mock())
+        _install_external_plugin("pypi:some-plugin", "===1.2.3", Mock(), None)
 
-        execute.assert_called_with(['pip', 'install', '--upgrade', '--pre', 'some-plugin===1.2.3'], ANY, shell=True,
-                                   error_file_name=ANY)
+        execute.assert_called_with(['pip', 'install', '--upgrade', 'some-plugin===1.2.3'], ANY, shell=False,
+                                   error_file_name=ANY, cwd=".")
+
+    @patch("pybuilder.pluginloader.read_file")
+    @patch("pybuilder.pluginloader.tempfile")
+    @patch("pybuilder.pluginloader.execute_command")
+    def test_should_install_plugin_with_vcs(self, execute, _, read_file):
+        read_file.return_value = ["no problems", "so far"]
+        execute.return_value = 0
+
+        _install_external_plugin("vcs:some-plugin URL", None, Mock(), None)
+
+        execute.assert_called_with(['pip', 'install', '--upgrade', 'some-plugin URL'], ANY, shell=False,
+                                   error_file_name=ANY, cwd=".")
+
+    @patch("pybuilder.pluginloader.read_file")
+    @patch("pybuilder.pluginloader.tempfile")
+    @patch("pybuilder.pluginloader.execute_command")
+    def test_should_install_plugin_with_vcs_and_version(self, execute, _, read_file):
+        read_file.return_value = ["no problems", "so far"]
+        execute.return_value = 0
+
+        _install_external_plugin("vcs:some-plugin URL", "===1.2.3", Mock(), None)
+
+        execute.assert_called_with(['pip', 'install', '--upgrade', 'some-plugin URL'], ANY, shell=False,
+                                   error_file_name=ANY, cwd=".")
 
     @patch("pybuilder.pluginloader.read_file")
     @patch("pybuilder.pluginloader.tempfile")
@@ -190,7 +243,16 @@ def test_should_raise_error_when_install_from_pypi_fails(self, execute, _, read_
         read_file.return_value = ["something", "went wrong"]
         execute.return_value = 1
 
-        self.assertRaises(MissingPluginException, _install_external_plugin, "pypi:some-plugin", None, Mock())
+        self.assertRaises(MissingPluginException, _install_external_plugin, "pypi:some-plugin", None, Mock(), None)
+
+    @patch("pybuilder.pluginloader.read_file")
+    @patch("pybuilder.pluginloader.tempfile")
+    @patch("pybuilder.pluginloader.execute_command")
+    def test_should_raise_error_when_install_from_vcs_fails(self, execute, _, read_file):
+        read_file.return_value = ["something", "went wrong"]
+        execute.return_value = 1
+
+        self.assertRaises(MissingPluginException, _install_external_plugin, "vcs:some VCS URL", None, Mock(), None)
 
 
 class BuiltinPluginLoaderTest(unittest.TestCase):
@@ -231,44 +293,44 @@ def test_should_import_plugin_when_requiring_plugin_and_plugin_is_found_as_built
 class DispatchingPluginLoaderTest(unittest.TestCase):
     def setUp(self):
         self.project = mock()
-        self.fist_delegatee = mock()
+        self.first_delegatee = mock()
         self.second_delegatee = mock()
 
         self.loader = DispatchingPluginLoader(
-            mock, self.fist_delegatee, self.second_delegatee)
+            mock, self.first_delegatee, self.second_delegatee)
 
-    def test_should_raise_exception_when_all_delgatees_raise_exception(self):
-        when(self.fist_delegatee).load_plugin(
-            self.project, "spam").thenRaise(MissingPluginException("spam"))
+    def test_should_raise_exception_when_all_delegatees_raise_exception(self):
+        when(self.first_delegatee).load_plugin(
+            self.project, "spam", None, None).thenRaise(MissingPluginException("spam"))
         when(self.second_delegatee).load_plugin(
-            self.project, "spam").thenRaise(MissingPluginException("spam"))
+            self.project, "spam", None, None).thenRaise(MissingPluginException("spam"))
 
         self.assertRaises(
             MissingPluginException, self.loader.load_plugin, self.project, "spam")
 
-        verify(self.fist_delegatee).load_plugin(self.project, "spam")
-        verify(self.second_delegatee).load_plugin(self.project, "spam")
+        verify(self.first_delegatee).load_plugin(self.project, "spam", None, None)
+        verify(self.second_delegatee).load_plugin(self.project, "spam", None, None)
 
-    def test_should_return_module_returned_by_second_loader_when_first_delgatee_raises_exception(self):
+    def test_should_return_module_returned_by_second_loader_when_first_delegatee_raises_exception(self):
         result = "result"
-        when(self.fist_delegatee).load_plugin(
-            self.project, "spam").thenRaise(MissingPluginException("spam"))
+        when(self.first_delegatee).load_plugin(
+            self.project, "spam", None, None).thenRaise(MissingPluginException("spam"))
         when(self.second_delegatee).load_plugin(
-            self.project, "spam").thenReturn(result)
+            self.project, "spam", None, None).thenReturn(result)
 
         self.assertEquals(
             result, self.loader.load_plugin(self.project, "spam"))
 
-        verify(self.fist_delegatee).load_plugin(self.project, "spam")
-        verify(self.second_delegatee).load_plugin(self.project, "spam")
+        verify(self.first_delegatee).load_plugin(self.project, "spam", None, None)
+        verify(self.second_delegatee).load_plugin(self.project, "spam", None, None)
 
-    def test_ensure_second_delegatee_is_not_trie_when_first_delegatee_loads_plugin(self):
+    def test_ensure_second_delegatee_will_not_try_when_first_delegatee_loads_plugin(self):
         result = "result"
-        when(self.fist_delegatee).load_plugin(
-            self.project, "spam").thenReturn(result)
+        when(self.first_delegatee).load_plugin(
+            self.project, "spam", None, None).thenReturn(result)
 
         self.assertEquals(
             result, self.loader.load_plugin(self.project, "spam"))
 
-        verify(self.fist_delegatee).load_plugin(self.project, "spam")
-        verify(self.second_delegatee, never).load_plugin(self.project, "spam")
+        verify(self.first_delegatee).load_plugin(self.project, "spam", None, None)
+        verify(self.second_delegatee, never).load_plugin(self.project, "spam", None, None)