diff --git a/.circleci/config.yml b/.circleci/config.yml index 23ad74de..daedfccb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -39,6 +39,8 @@ jobs: command: | pytest ci/integration_tests name: Integration tests + max_auto_reruns: 3 + auto_rerun_delay: 10s - run: command: | cd doc; make html @@ -86,9 +88,9 @@ workflows: parameters: python_ver: - "3.8" - - "3.10" - "3.11" - "3.12" + - "3.13" filters: tags: ignore: /.*/ diff --git a/Changelog.rst b/Changelog.rst index a73ab068..12009c30 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -1,6 +1,17 @@ Change Log ========== +2.15.0 +++++++ + +Changes +------- + +* Added compression support for all clients via `SSHClient(compress=True)`, `ParallelSSHClient(compress=True)` and + `HostConfig(compress=True)` - defaults to off. #252 +* Updated minimum `ssh2-python` and `ssh-python` requirements. + + 2.14.0 ++++++ diff --git a/ci/integration_tests/libssh2_clients/test_parallel_client.py b/ci/integration_tests/libssh2_clients/test_parallel_client.py index 5702bc2c..62aa9c6c 100644 --- a/ci/integration_tests/libssh2_clients/test_parallel_client.py +++ b/ci/integration_tests/libssh2_clients/test_parallel_client.py @@ -1934,5 +1934,17 @@ def test_no_ipv6(self): self.assertEqual(self.host, host_out.host) self.assertIsInstance(host_out.exception, NoIPv6AddressFoundError) + def test_compression_enabled(self): + client = ParallelSSHClient([self.host], port=self.port, pkey=self.user_key, num_retries=1, compress=True) + output = client.run_command(self.cmd, stop_on_errors=False) + client.join(output) + self.assertTrue(client._host_clients[0, self.host].compress) + expected_exit_code = 0 + expected_stdout = [self.resp] + stdout = list(output[0].stdout) + exit_code = output[0].exit_code + self.assertEqual(expected_exit_code, exit_code) + self.assertEqual(expected_stdout, stdout) + # TODO: # * password auth diff --git a/ci/integration_tests/libssh_clients/test_parallel_client.py b/ci/integration_tests/libssh_clients/test_parallel_client.py index 1907d39e..953cb7b2 100644 --- a/ci/integration_tests/libssh_clients/test_parallel_client.py +++ b/ci/integration_tests/libssh_clients/test_parallel_client.py @@ -515,6 +515,18 @@ def test_ipv6(self, gsocket): self.assertEqual(hosts[0], host_out.host) self.assertIsInstance(host_out.exception, TypeError) + def test_compression_enabled(self): + client = ParallelSSHClient([self.host], port=self.port, pkey=self.user_key, num_retries=1, compress=True) + output = client.run_command(self.cmd, stop_on_errors=False) + client.join(output) + self.assertTrue(client._host_clients[0, self.host].compress) + expected_exit_code = 0 + expected_stdout = [self.resp] + stdout = list(output[0].stdout) + exit_code = output[0].exit_code + self.assertEqual(expected_exit_code, exit_code) + self.assertEqual(expected_stdout, stdout) + # def test_multiple_run_command_timeout(self): # client = ParallelSSHClient([self.host], port=self.port, # pkey=self.user_key) diff --git a/pssh/clients/base/parallel.py b/pssh/clients/base/parallel.py index ffdb9433..3443ffa9 100644 --- a/pssh/clients/base/parallel.py +++ b/pssh/clients/base/parallel.py @@ -55,6 +55,7 @@ def __init__(self, hosts, user=None, password=None, port=None, pkey=None, gssapi_client_identity=None, gssapi_delegate_credentials=False, forward_ssh_agent=False, + compress=False, _auth_thread_pool=True, ): self.allow_agent = allow_agent @@ -86,6 +87,7 @@ def __init__(self, hosts, user=None, password=None, port=None, pkey=None, self.gssapi_server_identity = gssapi_server_identity self.gssapi_client_identity = gssapi_client_identity self.gssapi_delegate_credentials = gssapi_delegate_credentials + self.compress = compress self._auth_thread_pool = _auth_thread_pool self._check_host_config() diff --git a/pssh/clients/base/single.py b/pssh/clients/base/single.py index 05140aec..36a35444 100644 --- a/pssh/clients/base/single.py +++ b/pssh/clients/base/single.py @@ -226,6 +226,7 @@ def __init__(self, host, _auth_thread_pool=True, identity_auth=True, ipv6_only=False, + compress=False, ): super(PollMixIn, self).__init__() self._auth_thread_pool = _auth_thread_pool @@ -245,6 +246,7 @@ def __init__(self, host, self.identity_auth = identity_auth self._keepalive_greenlet = None self.ipv6_only = ipv6_only + self.compress = compress self._pool = Pool() self._init() diff --git a/pssh/clients/native/parallel.py b/pssh/clients/native/parallel.py index c8c879c2..cdd9d076 100644 --- a/pssh/clients/native/parallel.py +++ b/pssh/clients/native/parallel.py @@ -37,6 +37,7 @@ def __init__(self, hosts, user=None, password=None, port=22, pkey=None, forward_ssh_agent=False, keepalive_seconds=60, identity_auth=True, ipv6_only=False, + compress=False, ): """ :param hosts: Hosts to connect to @@ -115,6 +116,8 @@ def __init__(self, hosts, user=None, password=None, port=22, pkey=None, for the host(s) or raise NoIPv6AddressFoundError otherwise. Note this will disable connecting to an IPv4 address if an IP address is provided instead. :type ipv6_only: bool + :param compress: Enable/Disable compression on the client. Defaults to off. + :type compress: bool :raises: :py:class:`pssh.exceptions.PKeyFileError` on errors finding provided private key. @@ -126,6 +129,7 @@ def __init__(self, hosts, user=None, password=None, port=22, pkey=None, host_config=host_config, retry_delay=retry_delay, identity_auth=identity_auth, ipv6_only=ipv6_only, + compress=compress, ) self.proxy_host = proxy_host self.proxy_port = proxy_port @@ -232,6 +236,7 @@ def _make_ssh_client(self, host, cfg, _pkey_data): keepalive_seconds=cfg.keepalive_seconds or self.keepalive_seconds, identity_auth=cfg.identity_auth or self.identity_auth, ipv6_only=cfg.ipv6_only or self.ipv6_only, + compress=cfg.compress or self.compress, ) return _client diff --git a/pssh/clients/native/single.py b/pssh/clients/native/single.py index 15ae40f6..4e00769f 100644 --- a/pssh/clients/native/single.py +++ b/pssh/clients/native/single.py @@ -27,7 +27,7 @@ from ssh2.error_codes import LIBSSH2_ERROR_EAGAIN from ssh2.exceptions import SFTPHandleError, SFTPProtocolError, \ Timeout as SSH2Timeout -from ssh2.session import Session, LIBSSH2_SESSION_BLOCK_INBOUND, LIBSSH2_SESSION_BLOCK_OUTBOUND +from ssh2.session import Session, LIBSSH2_SESSION_BLOCK_INBOUND, LIBSSH2_SESSION_BLOCK_OUTBOUND, LIBSSH2_FLAG_COMPRESS from ssh2.sftp import LIBSSH2_FXF_READ, LIBSSH2_FXF_CREAT, LIBSSH2_FXF_WRITE, \ LIBSSH2_FXF_TRUNC, LIBSSH2_SFTP_S_IRUSR, LIBSSH2_SFTP_S_IRGRP, \ LIBSSH2_SFTP_S_IWUSR, LIBSSH2_SFTP_S_IXUSR, LIBSSH2_SFTP_S_IROTH, \ @@ -110,6 +110,7 @@ def __init__(self, host, keepalive_seconds=60, identity_auth=True, ipv6_only=False, + compress=False, ): """ :param host: Host name or IP to connect to. @@ -158,6 +159,8 @@ def __init__(self, host, for the host or raise NoIPv6AddressFoundError otherwise. Note this will disable connecting to an IPv4 address if an IP address is provided instead. :type ipv6_only: bool + :param compress: Enable/Disable compression on the client. Defaults to off. + :type compress: bool :raises: :py:class:`pssh.exceptions.PKeyFileError` on errors finding provided private key. @@ -182,6 +185,7 @@ def __init__(self, host, timeout=timeout, keepalive_seconds=keepalive_seconds, identity_auth=identity_auth, + compress=compress, ) proxy_host = '127.0.0.1' self._chan_stdout_lock = RLock() @@ -194,6 +198,7 @@ def __init__(self, host, proxy_host=proxy_host, proxy_port=proxy_port, identity_auth=identity_auth, ipv6_only=ipv6_only, + compress=compress, ) def _shell(self, channel): @@ -206,7 +211,9 @@ def _connect_proxy(self, proxy_host, proxy_port, proxy_pkey, allow_agent=True, timeout=None, forward_ssh_agent=False, keepalive_seconds=60, - identity_auth=True): + identity_auth=True, + compress=False, + ): assert isinstance(self.port, int) try: self._proxy_client = SSHClient( @@ -216,6 +223,7 @@ def _connect_proxy(self, proxy_host, proxy_port, proxy_pkey, timeout=timeout, forward_ssh_agent=forward_ssh_agent, identity_auth=identity_auth, keepalive_seconds=keepalive_seconds, + compress=compress, _auth_thread_pool=False) except Exception as ex: msg = "Proxy authentication failed. " \ @@ -263,6 +271,8 @@ def configure_keepalive(self): def _init_session(self, retries=1): self.session = Session() + if self.compress: + self.session.flag(LIBSSH2_FLAG_COMPRESS) if self.timeout: # libssh2 timeout is in ms diff --git a/pssh/clients/ssh/parallel.py b/pssh/clients/ssh/parallel.py index 7ab4aafd..d8e66343 100644 --- a/pssh/clients/ssh/parallel.py +++ b/pssh/clients/ssh/parallel.py @@ -40,6 +40,7 @@ def __init__(self, hosts, user=None, password=None, port=22, pkey=None, gssapi_delegate_credentials=False, identity_auth=True, ipv6_only=False, + compress=False, ): """ :param hosts: Hosts to connect to @@ -114,6 +115,8 @@ def __init__(self, hosts, user=None, password=None, port=22, pkey=None, for the host or raise NoIPv6AddressFoundError otherwise. Note this will disable connecting to an IPv4 address if an IP address is provided instead. :type ipv6_only: bool + :param compress: Enable/Disable compression on the client. Defaults to off. + :type compress: bool :raises: :py:class:`pssh.exceptions.PKeyFileError` on errors finding provided private key. @@ -125,6 +128,7 @@ def __init__(self, hosts, user=None, password=None, port=22, pkey=None, host_config=host_config, retry_delay=retry_delay, identity_auth=identity_auth, ipv6_only=ipv6_only, + compress=compress, ) self.pkey = _validate_pkey(pkey) self.cert_file = _validate_pkey_path(cert_file) @@ -228,5 +232,6 @@ def _make_ssh_client(self, host, cfg, _pkey_data): gssapi_client_identity=self.gssapi_client_identity, gssapi_delegate_credentials=self.gssapi_delegate_credentials, cert_file=cfg.cert_file, + compress=cfg.compress or self.compress, ) return _client diff --git a/pssh/clients/ssh/single.py b/pssh/clients/ssh/single.py index bc4cc2da..5b0be5b3 100644 --- a/pssh/clients/ssh/single.py +++ b/pssh/clients/ssh/single.py @@ -50,6 +50,7 @@ def __init__(self, host, gssapi_client_identity=None, gssapi_delegate_credentials=False, ipv6_only=False, + compress=False, _auth_thread_pool=True): """:param host: Host name or IP to connect to. :type host: str @@ -107,6 +108,8 @@ def __init__(self, host, for the host or raise NoIPv6AddressFoundError otherwise. Note this will disable connecting to an IPv4 address if an IP address is provided instead. :type ipv6_only: bool + :param compress: Enable/Disable compression on the client. Defaults to off. + :type compress: bool :raises: :py:class:`pssh.exceptions.PKeyFileError` on errors finding provided private key. @@ -124,6 +127,7 @@ def __init__(self, host, timeout=timeout, identity_auth=identity_auth, ipv6_only=ipv6_only, + compress=compress, ) def _disconnect(self): @@ -151,6 +155,8 @@ def _init_session(self, retries=1): self.session = Session() self.session.options_set(options.USER, self.user) self.session.options_set(options.HOST, self.host) + if self.compress: + self.session.options_set(options.COMPRESSION, "yes") self.session.options_set_port(self.port) if self.gssapi_server_identity: self.session.options_set( diff --git a/pssh/config.py b/pssh/config.py index 935b0d9a..d0c64eb4 100644 --- a/pssh/config.py +++ b/pssh/config.py @@ -29,7 +29,7 @@ class HostConfig(object): 'proxy_host', 'proxy_port', 'proxy_user', 'proxy_password', 'proxy_pkey', 'keepalive_seconds', 'ipv6_only', 'cert_file', 'auth_thread_pool', 'gssapi_auth', 'gssapi_server_identity', 'gssapi_client_identity', 'gssapi_delegate_credentials', - 'forward_ssh_agent', + 'forward_ssh_agent', 'compress', ) def __init__(self, user=None, port=None, password=None, private_key=None, @@ -46,6 +46,7 @@ def __init__(self, user=None, port=None, password=None, private_key=None, gssapi_client_identity=None, gssapi_delegate_credentials=False, forward_ssh_agent=False, + compress=False, ): """ :param user: Username to login as. @@ -99,6 +100,8 @@ def __init__(self, user=None, port=None, password=None, private_key=None, :param gssapi_delegate_credentials: Enable/disable server credentials delegation. (pssh.clients.ssh only) :type gssapi_delegate_credentials: bool + :param compress: Enable/Disable compression on the client. Defaults to off. + :type compress: bool """ self.user = user self.port = port @@ -124,6 +127,7 @@ def __init__(self, user=None, port=None, password=None, private_key=None, self.gssapi_server_identity = gssapi_server_identity self.gssapi_client_identity = gssapi_client_identity self.gssapi_delegate_credentials = gssapi_delegate_credentials + self.compress = compress self._sanity_checks() def _sanity_checks(self): @@ -181,3 +185,5 @@ def _sanity_checks(self): raise ValueError("GSSAPI client identity %s is not a string", self.gssapi_client_identity) if self.gssapi_delegate_credentials is not None and not isinstance(self.gssapi_delegate_credentials, bool): raise ValueError("GSSAPI delegate credentials %s is not a bool", self.gssapi_delegate_credentials) + if self.compress is not None and not isinstance(self.compress, bool): + raise ValueError("Compress %s is not a bool", self.compress) diff --git a/requirements.txt b/requirements.txt index eaba1f61..ea7fed43 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ gevent>=1.3.0 -ssh2-python>=1.1.0 -ssh-python>=1.1.0 +ssh2-python>=1.2.0 +ssh-python>=1.2.0 diff --git a/tests/test_host_config.py b/tests/test_host_config.py index 7327fb78..4bb10e9a 100644 --- a/tests/test_host_config.py +++ b/tests/test_host_config.py @@ -43,6 +43,7 @@ def test_host_config_entries(self): gssapi_server_identity = 'some_id' gssapi_client_identity = 'some_id' gssapi_delegate_credentials = True + compress = True cfg = HostConfig( user=user, port=port, password=password, alias=alias, private_key=private_key, allow_agent=allow_agent, num_retries=num_retries, retry_delay=retry_delay, @@ -56,6 +57,7 @@ def test_host_config_entries(self): gssapi_server_identity=gssapi_server_identity, gssapi_client_identity=gssapi_client_identity, gssapi_delegate_credentials=gssapi_delegate_credentials, + compress=compress, ) self.assertEqual(cfg.user, user) self.assertEqual(cfg.port, port) @@ -76,6 +78,7 @@ def test_host_config_entries(self): self.assertEqual(cfg.gssapi_server_identity, gssapi_server_identity) self.assertEqual(cfg.gssapi_client_identity, gssapi_client_identity) self.assertEqual(cfg.gssapi_delegate_credentials, gssapi_delegate_credentials) + self.assertEqual(cfg.compress, compress) def test_host_config_bad_entries(self): self.assertRaises(ValueError, HostConfig, user=22) @@ -102,3 +105,4 @@ def test_host_config_bad_entries(self): self.assertRaises(ValueError, HostConfig, gssapi_server_identity=1) self.assertRaises(ValueError, HostConfig, gssapi_client_identity=1) self.assertRaises(ValueError, HostConfig, gssapi_delegate_credentials='') + self.assertRaises(ValueError, HostConfig, compress='')