Merge pull request #345 from arcivanov/issue_341

Support for pip --trusted-host argument enhancement

src/main/python/pybuilder/pip_utils.py

@@ -60,9 +60,9 @@ def build_dependency_version_string(mixed):
             raise ValueError("'%s' must be either PEP 0440 version or a version specifier set")
 
 
-def pip_install(install_targets, index_url=None, extra_index_url=None, upgrade=False, insecure_installs=None,
-                force_reinstall=False, target_dir=None, verbose=False, logger=None, outfile_name=None,
-                error_file_name=None, env=None, cwd=None):
+def pip_install(install_targets, index_url=None, extra_index_url=None, upgrade=False,
+                insecure_installs=None, force_reinstall=False, target_dir=None, verbose=False, logger=None,
+                outfile_name=None, error_file_name=None, env=None, cwd=None, trusted_host=None):
     pip_command_line = list()
     pip_command_line.extend(PIP_EXEC_STANZA)
     pip_command_line.append("install")
@@ -72,7 +72,8 @@ def pip_install(install_targets, index_url=None, extra_index_url=None, upgrade=F
                                                       insecure_installs,
                                                       force_reinstall,
                                                       target_dir,
-                                                      verbose
+                                                      verbose,
+                                                      trusted_host
                                                       ))
     for install_target in as_list(install_targets):
         pip_command_line.extend(as_pip_install_target(install_target))
@@ -87,14 +88,23 @@ def pip_install(install_targets, index_url=None, extra_index_url=None, upgrade=F
 
 
 def build_pip_install_options(index_url=None, extra_index_url=None, upgrade=False, insecure_installs=None,
-                              force_reinstall=False, target_dir=None, verbose=False):
+                              force_reinstall=False, target_dir=None, verbose=False, trusted_host=None):
     options = []
     if index_url:
         options.append("--index-url")
         options.append(index_url)
-        if extra_index_url:
+
+    if extra_index_url:
+        extra_index_urls = as_list(extra_index_url)
+        for url in extra_index_urls:
             options.append("--extra-index-url")
-            options.append(extra_index_url)
+            options.append(url)
+
+    if trusted_host:
+        trusted_hosts = as_list(trusted_host)
+        for host in trusted_hosts:
+            options.append("--trusted-host")
+            options.append(host)
 
     if upgrade:
         options.append("--upgrade")

src/main/python/pybuilder/pluginloader.py

@@ -97,10 +97,7 @@ def load_plugin(self, project, name, version=None, plugin_module_name=None):
         if update_plugin or force_reinstall:
             self.logger.info("Downloading or updating plugin {0}".format(display_name))
             try:
-                _install_external_plugin(name, version, self.logger, plugin_module_name,
-                                         project.get_property("install_dependencies_index_url"),
-                                         project.get_property("install_dependencies_extra_index_url"),
-                                         update_plugin,
+                _install_external_plugin(project, name, version, self.logger, plugin_module_name, update_plugin,
                                          force_reinstall)
                 self.logger.info("Installed or updated plugin {0}.".format(display_name))
             except MissingPluginException as e:
@@ -118,9 +115,7 @@ def load_plugin(self, project, name, version=None, plugin_module_name=None):
         # We have failed to update or to load a plugin without a previous installation
         self.logger.info("Downloading plugin {0}".format(display_name))
         try:
-            _install_external_plugin(name, version, self.logger, plugin_module_name,
-                                     project.get_property("install_dependencies_index_url"),
-                                     project.get_property("install_dependencies_extra_index_url"))
+            _install_external_plugin(project, name, version, self.logger, plugin_module_name)
             self.logger.info("Installed plugin {0}.".format(display_name))
         except MissingPluginException as e:
             self.logger.error("Could not install plugin {0}: {1}.".format(display_name, e))
@@ -162,8 +157,7 @@ def load_plugin(self, project, name, version=None, plugin_module_name=None):
                                          "no plugin loader was able to load the plugin specified")
 
 
-def _install_external_plugin(name, version, logger, plugin_module_name, index_url=None, extra_index_url=None,
-                             upgrade=False, force_reinstall=False):
+def _install_external_plugin(project, name, version, logger, plugin_module_name, upgrade=False, force_reinstall=False):
     if not name.startswith(PYPI_PLUGIN_PROTOCOL) and not name.startswith(VCS_PLUGIN_PROTOCOL):
         message = "Only plugins starting with '{0}' are currently supported"
         raise MissingPluginException(name, message.format((PYPI_PLUGIN_PROTOCOL, VCS_PLUGIN_PROTOCOL)))
@@ -180,8 +174,9 @@ def _install_external_plugin(name, version, logger, plugin_module_name, index_ur
     with tempfile.NamedTemporaryFile(delete=True) as log_file:
         log_file_name = log_file.name
         result = pip_install(pip_package,
-                             index_url=index_url,
-                             extra_index_url=extra_index_url,
+                             index_url=project.get_property("install_dependencies_index_url"),
+                             extra_index_url=project.get_property("install_dependencies_extra_index_url"),
+                             trusted_host=project.get_property("install_dependencies_trusted_host"),
                              upgrade=upgrade,
                              force_reinstall=force_reinstall,
                              logger=logger,

src/main/python/pybuilder/plugins/python/install_dependencies_plugin.py

@@ -45,6 +45,7 @@ def initialize_install_dependencies_plugin(project):
     project.set_property_if_unset("install_dependencies_index_url", None)
     project.set_property_if_unset("install_dependencies_local_mapping", {})
     project.set_property_if_unset("install_dependencies_extra_index_url", None)
+    project.set_property_if_unset("install_dependencies_trusted_host", None)
     project.set_property_if_unset("install_dependencies_upgrade", False)
     project.set_property_if_unset("install_dependencies_insecure_installation", [])
 
@@ -112,7 +113,8 @@ def install_dependency(logger, project, dependency):
                                                           "install_dependencies_insecure_installation"),
                                                       True if url else False,
                                                       target_dir,
-                                                      project.get_property("verbose")
+                                                      project.get_property("verbose"),
+                                                      project.get_property("install_dependencies_trusted_host")
                                                       ))
     pip_command_line.extend(as_pip_install_target(dependency))
     logger.debug("Invoking pip: %s", pip_command_line)

src/unittest/python/pip_utils_tests.py

@@ -72,9 +72,15 @@ def test_get_package_version(self):
     def test_build_pip_install_options(self):
         self.assertEquals(pip_utils.build_pip_install_options(), [])
         self.assertEquals(pip_utils.build_pip_install_options(index_url="foo"), ["--index-url", "foo"])
-        self.assertEquals(pip_utils.build_pip_install_options(extra_index_url="foo"), [])
+        self.assertEquals(pip_utils.build_pip_install_options(extra_index_url="foo"), ["--extra-index-url", "foo"])
         self.assertEquals(pip_utils.build_pip_install_options(index_url="foo", extra_index_url="bar"),
                           ["--index-url", "foo", "--extra-index-url", "bar"])
+        self.assertEquals(pip_utils.build_pip_install_options(extra_index_url=("foo", "bar")),
+                          ["--extra-index-url", "foo", "--extra-index-url", "bar"])
+        self.assertEquals(pip_utils.build_pip_install_options(trusted_host="foo"),
+                          ["--trusted-host", "foo"])
+        self.assertEquals(pip_utils.build_pip_install_options(trusted_host=("foo", "bar")),
+                          ["--trusted-host", "foo", "--trusted-host", "bar"])
         self.assertEquals(pip_utils.build_pip_install_options(upgrade=True), ["--upgrade"])
         self.assertEquals(pip_utils.build_pip_install_options(verbose=True), ["--verbose"])
         self.assertEquals(pip_utils.build_pip_install_options(force_reinstall=True), ["--force-reinstall"])

src/unittest/python/pluginloader_tests.py

@@ -82,7 +82,7 @@ def test_should_download_module_from_pypi(self, install, load):
         load.side_effect = (MissingPluginException("external_plugin"), Mock())
         DownloadingPluginLoader(logger).load_plugin(project, "pypi:external_plugin")
 
-        install.assert_called_with("pypi:external_plugin", None, logger, None, "index_url", "extra_index_url")
+        install.assert_called_with(project, "pypi:external_plugin", None, logger, None)
 
     @patch("pybuilder.pluginloader._load_plugin")
     @patch("pybuilder.pluginloader._install_external_plugin")
@@ -171,7 +171,7 @@ def test_should_force_reinstall_vcs_plugin_before_first_loading_attempt(self, in
 
         self.assertEquals(load.return_value, downloader.load_plugin(project, "vcs:spam", plugin_module_name="spam"))
 
-        install.assert_called_with("vcs:spam", None, downloader.logger, "spam", ANY, ANY, False, True)
+        install.assert_called_with(project, "vcs:spam", None, downloader.logger, "spam", False, True)
         self.assertEquals(install.call_count, 1)
 
     @patch("pybuilder.pluginloader._load_plugin")
@@ -183,7 +183,7 @@ def test_should_update_pypi_plugin_with_non_exact_version_before_first_loading_a
 
         self.assertEquals(load.return_value, downloader.load_plugin(project, "pypi:spam", ">1.2"))
 
-        install.assert_called_with("pypi:spam", ">1.2", downloader.logger, None, ANY, ANY, True, False)
+        install.assert_called_with(project, "pypi:spam", ">1.2", downloader.logger, None, True, False)
         self.assertEquals(install.call_count, 1)
 
     @patch("pybuilder.pluginloader._load_plugin")
@@ -196,7 +196,7 @@ def test_should_update_pypi_plugin_with_compound_non_exact_version_before_first_
 
         self.assertEquals(load.return_value, downloader.load_plugin(project, "pypi:spam", ">1.2,==1.4"))
 
-        install.assert_called_with("pypi:spam", ">1.2,==1.4", downloader.logger, None, ANY, ANY, True, False)
+        install.assert_called_with(project, "pypi:spam", ">1.2,==1.4", downloader.logger, None, True, False)
         self.assertEquals(install.call_count, 1)
 
     @patch("pybuilder.pluginloader._load_plugin")
@@ -209,13 +209,13 @@ def test_should_not_update_pypi_plugin_with_exact_version_before_first_loading_a
 
         self.assertEquals(plugin, downloader.load_plugin(project, "pypi:spam", "===1.4"))
 
-        install.assert_called_with("pypi:spam", "===1.4", downloader.logger, None, ANY, ANY)
+        install.assert_called_with(project, "pypi:spam", "===1.4", downloader.logger, None)
         self.assertEquals(install.call_count, 1)
 
 
 class InstallExternalPluginTests(unittest.TestCase):
     def test_should_raise_error_when_protocol_is_invalid(self):
-        self.assertRaises(MissingPluginException, _install_external_plugin, "some-plugin", None, Mock(), None)
+        self.assertRaises(MissingPluginException, _install_external_plugin, Mock(), "some-plugin", None, Mock(), None)
 
     @patch("pybuilder.pluginloader.read_file")
     @patch("pybuilder.pluginloader.tempfile")
@@ -224,10 +224,11 @@ def test_should_install_plugin(self, execute, _, read_file):
         read_file.return_value = ["no problems", "so far"]
         execute.return_value = 0
 
-        _install_external_plugin("pypi:some-plugin", None, Mock(), None)
+        _install_external_plugin(Mock(), "pypi:some-plugin", None, Mock(), None)
 
-        execute.assert_called_with(PIP_EXEC_STANZA + ['install', 'some-plugin'], shell=False,
-                                   outfile_name=ANY, error_file_name=ANY, cwd=".", env=ANY)
+        execute.assert_called_with(
+            PIP_EXEC_STANZA + ['install', '--index-url', ANY, '--extra-index-url', ANY, '--trusted-host', ANY,
+                               'some-plugin'], shell=False, outfile_name=ANY, error_file_name=ANY, cwd=".", env=ANY)
 
     @patch("pybuilder.pluginloader.read_file")
     @patch("pybuilder.pluginloader.tempfile")
@@ -236,10 +237,12 @@ def test_should_install_plugin_with_version(self, execute, _, read_file):
         read_file.return_value = ["no problems", "so far"]
         execute.return_value = 0
 
-        _install_external_plugin("pypi:some-plugin", "===1.2.3", Mock(), None)
+        _install_external_plugin(Mock(), "pypi:some-plugin", "===1.2.3", Mock(), None)
 
-        execute.assert_called_with(PIP_EXEC_STANZA + ['install', '--upgrade', 'some-plugin===1.2.3'], shell=False,
-                                   outfile_name=ANY, error_file_name=ANY, cwd=".", env=ANY)
+        execute.assert_called_with(
+            PIP_EXEC_STANZA + ['install', '--index-url', ANY, '--extra-index-url', ANY, '--trusted-host', ANY,
+                               '--upgrade', 'some-plugin===1.2.3'], shell=False, outfile_name=ANY, error_file_name=ANY,
+            cwd=".", env=ANY)
 
     @patch("pybuilder.pluginloader.read_file")
     @patch("pybuilder.pluginloader.tempfile")
@@ -248,10 +251,12 @@ def test_should_install_plugin_with_vcs(self, execute, _, read_file):
         read_file.return_value = ["no problems", "so far"]
         execute.return_value = 0
 
-        _install_external_plugin("vcs:some-plugin URL", None, Mock(), None)
+        _install_external_plugin(Mock(), "vcs:some-plugin URL", None, Mock(), None)
 
-        execute.assert_called_with(PIP_EXEC_STANZA + ['install', '--force-reinstall', 'some-plugin URL'], shell=False,
-                                   outfile_name=ANY, error_file_name=ANY, cwd=".", env=ANY)
+        execute.assert_called_with(
+            PIP_EXEC_STANZA + ['install', '--index-url', ANY, '--extra-index-url', ANY, '--trusted-host', ANY,
+                               '--force-reinstall', 'some-plugin URL'], shell=False, outfile_name=ANY,
+            error_file_name=ANY, cwd=".", env=ANY)
 
     @patch("pybuilder.pluginloader.read_file")
     @patch("pybuilder.pluginloader.tempfile")
@@ -260,10 +265,12 @@ def test_should_install_plugin_with_vcs_and_version(self, execute, _, read_file)
         read_file.return_value = ["no problems", "so far"]
         execute.return_value = 0
 
-        _install_external_plugin("vcs:some-plugin URL", "===1.2.3", Mock(), None)
+        _install_external_plugin(Mock(), "vcs:some-plugin URL", "===1.2.3", Mock(), None)
 
-        execute.assert_called_with(PIP_EXEC_STANZA + ['install', '--force-reinstall', 'some-plugin URL'], shell=False,
-                                   outfile_name=ANY, error_file_name=ANY, cwd=".", env=ANY)
+        execute.assert_called_with(
+            PIP_EXEC_STANZA + ['install', '--index-url', ANY, '--extra-index-url', ANY, '--trusted-host', ANY,
+                               '--force-reinstall', 'some-plugin URL'], shell=False, outfile_name=ANY,
+            error_file_name=ANY, cwd=".", env=ANY)
 
     @patch("pybuilder.pluginloader.read_file")
     @patch("pybuilder.pluginloader.tempfile")
@@ -272,7 +279,7 @@ def test_should_raise_error_when_install_from_pypi_fails(self, execute, _, read_
         read_file.return_value = ["something", "went wrong"]
         execute.return_value = 1
 
-        self.assertRaises(MissingPluginException, _install_external_plugin, "pypi:some-plugin", None, Mock(), None)
+        self.assertRaises(MissingPluginException, _install_external_plugin, Mock(), "pypi:some-plugin", None, Mock(), None)
 
     @patch("pybuilder.pluginloader.read_file")
     @patch("pybuilder.pluginloader.tempfile")
@@ -281,7 +288,7 @@ def test_should_raise_error_when_install_from_vcs_fails(self, execute, _, read_f
         read_file.return_value = ["something", "went wrong"]
         execute.return_value = 1
 
-        self.assertRaises(MissingPluginException, _install_external_plugin, "vcs:some VCS URL", None, Mock(), None)
+        self.assertRaises(MissingPluginException, _install_external_plugin, Mock(), "vcs:some VCS URL", None, Mock(), None)
 
 
 class BuiltinPluginLoaderTest(unittest.TestCase):

src/unittest/python/plugins/python/install_dependencies_plugin_tests.py

@@ -118,16 +118,17 @@ def test_should_install_dependency_using_custom_index_url(self):
             PIP_EXEC_STANZA + ["install", "--index-url", "some_index_url", 'spam'], any_value(), env=any_value(),
             shell=False)
 
-    def test_should_not_use_extra_index_url_when_index_url_is_not_set(self):
-        self.project.set_property("install_dependencies_extra_index_url", "some_index_url")
+    def test_should_use_extra_index_url_when_index_url_is_not_set(self):
+        self.project.set_property("install_dependencies_extra_index_url", "some_extra_index_url")
         dependency = Dependency("spam")
 
         install_dependency(self.logger, self.project, dependency)
 
         verify(pybuilder.plugins.python.install_dependencies_plugin).execute_command(
-            PIP_EXEC_STANZA + ["install", 'spam'], any_value(), env=any_value(), shell=False)
+            PIP_EXEC_STANZA + ["install", "--extra-index-url", "some_extra_index_url", 'spam'], any_value(),
+            env=any_value(), shell=False)
 
-    def test_should_not_use_index_and_extra_index_url_when_index_and_extra_index_url_are_set(self):
+    def test_should_use_index_and_extra_index_url_when_index_and_extra_index_url_are_set(self):
         self.project.set_property("install_dependencies_index_url", "some_index_url")
         self.project.set_property("install_dependencies_extra_index_url", "some_extra_index_url")
         dependency = Dependency("spam")