Host config (#342)

* Updated clients for getting all configuration entries from host config
* Updated changelog, documentation
* Updated tests
* Added bytes support to proxy_pkey, added test - resolves #338
* Updated RTD configuration
* Updated setup.cfg
* Updated CircleCI cfg
* Updated copyright notices

Author: pkittenis

.circleci/config.yml

@@ -39,10 +39,7 @@ jobs:
           command: |
             set -x
             eval "$(ssh-agent -s)"
-            pytest --cov-append --cov=pssh tests/test_output.py tests/test_utils.py tests/test_host_config.py
-            pytest --reruns 10 --cov-append --cov=pssh tests/native/test_tunnel.py tests/native/test_agent.py
-            pytest --reruns 5 --cov-append --cov=pssh tests/native/test_*_client.py
-            pytest --reruns 5 --cov-append --cov=pssh tests/ssh
+            pytest
           name: Integration tests
       - run:
           command: |
@@ -97,8 +94,9 @@ workflows:
             parameters:
               python_ver:
                 - "3.6"
-                - "3.7"
                 - "3.8"
+                - "3.9"
+                - "3.10"
           filters:
             tags:
               ignore: /.*/

.environment.yml

@@ -1,4 +1,9 @@
-conda:
-  file: .environment.yml
+version: 2
+build:
+  apt_packages:
+    - cmake
+    - openssl
 python:
-  pip_install: true
+  install:
+    - method: pip
+      path: .

.readthedocs.yml

@@ -1,6 +1,16 @@
 Change Log
 ============
 
+2.10.0
+++++++
+
+Changes
+-------
+
+* All client configuration can now be provided via ``HostConfig`` on parallel clients.
+* ``proxy_pkey`` can now also be provided as bytes for proxy authentication from in-memory private key data - #338
+* Removed deprecated since ``2.0.0`` dictionary support for ``host_config`` entries.
+
 2.9.1
 +++++
 
@@ -18,7 +28,6 @@ Changes
 * ``pssh.exceptions.ConnectionError`` is now the same as built-in ``ConnectionError`` and deprecated - to be removed.
 * Clients now attempt to connect with all addresses in DNS list. In the case where an address refuses connection,
   other available addresses are attempted without delay.
-
   For example where a host resolves to both IPv4 and v6 addresses while only one address is
   accepting connections, or multiple v4/v6 addresses where only some are accepting connections.
 * Connection actively refused error is no longer subject to retries.

Changelog.rst

@@ -352,7 +352,7 @@ In the above example, the client is configured to connect to hostname ``localhos
 
 When using ``host_config``, the number of ``HostConfig`` entries must match the number of hosts in ``client.hosts``. An exception is raised on client initialisation if not.
 
-As of `2.2.0`, proxy configuration can also be provided in ``HostConfig``.
+As of `2.10.0`, all client configuration can be provided in ``HostConfig``.
 
 .. _per-host-cmds:
 

doc/advanced.rst

@@ -1,6 +1,6 @@
 # This file is part of parallel-ssh.
 #
-# Copyright (C) 2014-2020 Panos Kittenis.
+# Copyright (C) 2014-2022 Panos Kittenis and contributors.
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public

pssh/__init__.py

@@ -1,6 +1,6 @@
 # This file is part of parallel-ssh.
 #
-# Copyright (C) 2014-2020 Panos Kittenis.
+# Copyright (C) 2014-2022 Panos Kittenis and contributors.
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public

pssh/clients/__init__.py

@@ -1,6 +1,6 @@
 # This file is part of parallel-ssh.
 #
-# Copyright (C) 2014-2020 Panos Kittenis.
+# Copyright (C) 2014-2022 Panos Kittenis and contributors.
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -20,15 +20,15 @@
 import logging
 
 import gevent.pool
-
 from gevent import joinall, spawn, Timeout as GTimeout
 from gevent.hub import Hub
 
+from ..common import _validate_pkey_path
+from ...config import HostConfig
 from ...constants import DEFAULT_RETRIES, RETRY_DELAY
-from ...exceptions import HostArgumentError, Timeout, ShellError
+from ...exceptions import HostArgumentError, Timeout, ShellError, HostConfigError
 from ...output import HostOutput
 
-
 Hub.NOT_ERROR = (Exception,)
 logger = logging.getLogger(__name__)
 
@@ -43,6 +43,19 @@ def __init__(self, hosts, user=None, password=None, port=None, pkey=None,
                  host_config=None, retry_delay=RETRY_DELAY,
                  identity_auth=True,
                  ipv6_only=False,
+                 proxy_host=None,
+                 proxy_port=None,
+                 proxy_user=None,
+                 proxy_password=None,
+                 proxy_pkey=None,
+                 keepalive_seconds=None,
+                 cert_file=None,
+                 gssapi_auth=False,
+                 gssapi_server_identity=None,
+                 gssapi_client_identity=None,
+                 gssapi_delegate_credentials=False,
+                 forward_ssh_agent=False,
+                 _auth_thread_pool=True,
                  ):
         self.allow_agent = allow_agent
         self.pool_size = pool_size
@@ -60,6 +73,19 @@ def __init__(self, hosts, user=None, password=None, port=None, pkey=None,
         self.cmds = None
         self.identity_auth = identity_auth
         self.ipv6_only = ipv6_only
+        self.proxy_host = proxy_host
+        self.proxy_port = proxy_port
+        self.proxy_user = proxy_user
+        self.proxy_password = proxy_password
+        self.proxy_pkey = proxy_pkey
+        self.keepalive_seconds = keepalive_seconds
+        self.cert_file = cert_file
+        self.forward_ssh_agent = forward_ssh_agent
+        self.gssapi_auth = gssapi_auth
+        self.gssapi_server_identity = gssapi_server_identity
+        self.gssapi_client_identity = gssapi_client_identity
+        self.gssapi_delegate_credentials = gssapi_delegate_credentials
+        self._auth_thread_pool = _auth_thread_pool
         self._check_host_config()
 
     def _validate_hosts(self, _hosts):
@@ -100,7 +126,7 @@ def _check_host_config(self):
     def _open_shell(self, host_i, host,
                     encoding='utf-8', read_timeout=None):
         try:
-            _client = self._make_ssh_client(host_i, host)
+            _client = self._get_ssh_client(host_i, host)
             shell = _client.open_shell(
                 encoding=encoding, read_timeout=read_timeout)
             return shell
@@ -230,36 +256,37 @@ def get_last_output(self, cmds=None):
         return self._get_output_from_cmds(
             cmds, raise_error=False)
 
-    def _get_host_config_values(self, host_i, host):
+    def _get_host_config(self, host_i, host):
         if self.host_config is None:
-            return self.user, self.port, self.password, self.pkey, \
-                getattr(self, 'proxy_host', None), \
-                getattr(self, 'proxy_port', None), getattr(self, 'proxy_user', None), \
-                getattr(self, 'proxy_password', None), getattr(self, 'proxy_pkey', None)
-        elif isinstance(self.host_config, list):
-            config = self.host_config[host_i]
-            return config.user or self.user, config.port or self.port, \
-                config.password or self.password, config.private_key or self.pkey, \
-                config.proxy_host or getattr(self, 'proxy_host', None), \
-                config.proxy_port or getattr(self, 'proxy_port', None), \
-                config.proxy_user or getattr(self, 'proxy_user', None), \
-                config.proxy_password or getattr(self, 'proxy_password', None), \
-                config.proxy_pkey or getattr(self, 'proxy_pkey', None)
-        elif isinstance(self.host_config, dict):
-            _user = self.host_config.get(host, {}).get('user', self.user)
-            _port = self.host_config.get(host, {}).get('port', self.port)
-            _password = self.host_config.get(host, {}).get(
-                'password', self.password)
-            _pkey = self.host_config.get(host, {}).get('private_key', self.pkey)
-            return _user, _port, _password, _pkey, None, None, None, None, None
+            config = HostConfig(
+                user=self.user, port=self.port, password=self.password, private_key=self.pkey,
+                allow_agent=self.allow_agent, num_retries=self.num_retries, retry_delay=self.retry_delay,
+                timeout=self.timeout, identity_auth=self.identity_auth, proxy_host=self.proxy_host,
+                proxy_port=self.proxy_port, proxy_user=self.proxy_user, proxy_password=self.proxy_password,
+                proxy_pkey=self.proxy_pkey,
+                keepalive_seconds=self.keepalive_seconds,
+                ipv6_only=self.ipv6_only,
+                cert_file=self.cert_file,
+                forward_ssh_agent=self.forward_ssh_agent,
+                gssapi_auth=self.gssapi_auth,
+                gssapi_server_identity=self.gssapi_server_identity,
+                gssapi_client_identity=self.gssapi_client_identity,
+                gssapi_delegate_credentials=self.gssapi_delegate_credentials,
+            )
+            return config
+        elif not isinstance(self.host_config, list):
+            raise HostConfigError("Host configuration of type %s is invalid - valid types are list[HostConfig]",
+                                  type(self.host_config))
+        config = self.host_config[host_i]
+        return config
 
     def _run_command(self, host_i, host, command, sudo=False, user=None,
                      shell=None, use_pty=False,
                      encoding='utf-8', read_timeout=None):
         """Make SSHClient if needed, run command on host"""
         logger.debug("_run_command with read timeout %s", read_timeout)
         try:
-            _client = self._make_ssh_client(host_i, host)
+            _client = self._get_ssh_client(host_i, host)
             host_out = _client.run_command(
                 command, sudo=sudo, user=user, shell=shell,
                 use_pty=use_pty, encoding=encoding, read_timeout=read_timeout)
@@ -283,7 +310,7 @@ def connect_auth(self):
         :returns: list of greenlets to ``joinall`` with.
         :rtype: list(:py:mod:`gevent.greenlet.Greenlet`)
         """
-        cmds = [spawn(self._make_ssh_client, i, host) for i, host in enumerate(self.hosts)]
+        cmds = [spawn(self._get_ssh_client, i, host) for i, host in enumerate(self.hosts)]
         return cmds
 
     def _consume_output(self, stdout, stderr):
@@ -429,7 +456,7 @@ def copy_file(self, local_file, remote_file, recurse=False, copy_args=None):
 
     def _copy_file(self, host_i, host, local_file, remote_file, recurse=False):
         """Make sftp client, copy file"""
-        client = self._make_ssh_client(host_i, host)
+        client = self._get_ssh_client(host_i, host)
         return client.copy_file(
             local_file, remote_file, recurse=recurse)
 
@@ -512,7 +539,7 @@ def copy_remote_file(self, remote_file, local_file, recurse=False,
     def _copy_remote_file(self, host_i, host, remote_file, local_file, recurse,
                           **kwargs):
         """Make sftp client, copy file to local"""
-        client = self._make_ssh_client(host_i, host)
+        client = self._get_ssh_client(host_i, host)
         return client.copy_remote_file(
             remote_file, local_file, recurse=recurse, **kwargs)
 
@@ -522,5 +549,26 @@ def _handle_greenlet_exc(self, func, host, *args, **kwargs):
         except Exception as ex:
             raise ex
 
-    def _make_ssh_client(self, host_i, host):
+    def _get_ssh_client(self, host_i, host):
+        logger.debug("Make client request for host %s, (host_i, host) in clients: %s",
+                     host, (host_i, host) in self._host_clients)
+        _client = self._host_clients.get((host_i, host))
+        if _client is not None:
+            return _client
+        cfg = self._get_host_config(host_i, host)
+        _pkey = self.pkey if cfg.private_key is None else cfg.private_key
+        _pkey_data = self._load_pkey_data(_pkey)
+        _client = self._make_ssh_client(host, cfg, _pkey_data)
+        self._host_clients[(host_i, host)] = _client
+        return _client
+
+    def _load_pkey_data(self, _pkey):
+        if isinstance(_pkey, str):
+            _validate_pkey_path(_pkey)
+            with open(_pkey, 'rb') as fh:
+                _pkey_data = fh.read()
+            return _pkey_data
+        return _pkey
+
+    def _make_ssh_client(self, host, cfg, _pkey_data):
         raise NotImplementedError

pssh/clients/base/parallel.py

@@ -1,6 +1,6 @@
 # This file is part of parallel-ssh.
 #
-# Copyright (C) 2014-2020 Panos Kittenis.
+# Copyright (C) 2014-2022 Panos Kittenis and contributors.
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public

pssh/clients/base/single.py

@@ -1,6 +1,6 @@
 # This file is part of parallel-ssh.
 #
-# Copyright (C) 2014-2020 Panos Kittenis.
+# Copyright (C) 2014-2022 Panos Kittenis and contributors.
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public

pssh/clients/common.py

@@ -1,6 +1,6 @@
 # This file is part of parallel-ssh.
 #
-# Copyright (C) 2014-2020 Panos Kittenis.
+# Copyright (C) 2014-2022 Panos Kittenis and contributors.
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public