From 94ff256085a09861b595f47e865aab951c9d4b46 Mon Sep 17 00:00:00 2001 From: Tim Sampson Date: Fri, 15 Mar 2019 10:54:38 +0200 Subject: [PATCH 1/7] native.scp_send: don't try to recreate remote dir if don't have sftp --- pssh/clients/native/single.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/pssh/clients/native/single.py b/pssh/clients/native/single.py index 9757fb40..1d09559a 100644 --- a/pssh/clients/native/single.py +++ b/pssh/clients/native/single.py @@ -771,13 +771,14 @@ def scp_send(self, local_file, remote_file, recurse=False, sftp=None): elif os.path.isdir(local_file) and not recurse: raise ValueError("Recurse must be True if local_file is a " "directory.") - destination = self._remote_paths_split(remote_file) - if destination is not None: - sftp = self._make_sftp() if sftp is None else sftp - try: - self._eagain(sftp.stat, destination) - except (SFTPHandleError, SFTPProtocolError): - self.mkdir(sftp, destination) + if recurse: + destination = self._remote_paths_split(remote_file) + if destination is not None: + sftp = self._make_sftp() if sftp is None else sftp + try: + self._eagain(sftp.stat, destination) + except (SFTPHandleError, SFTPProtocolError): + self.mkdir(sftp, destination) self._scp_send(local_file, remote_file) logger.info("SCP local file %s to remote destination %s:%s", local_file, self.host, remote_file) From 18cdf6fc8efcae7495f2bdafbeed0a9cf5b1922a Mon Sep 17 00:00:00 2001 From: Casey Peel Date: Mon, 2 Dec 2019 17:35:53 -0800 Subject: [PATCH 2/7] Add tests for updated scp-without-sftp fix --- tests/test_native_parallel_client.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/test_native_parallel_client.py b/tests/test_native_parallel_client.py index 0a110d5b..18f5d264 100644 --- a/tests/test_native_parallel_client.py +++ b/tests/test_native_parallel_client.py @@ -1269,6 +1269,10 @@ def test_file_read_no_timeout(self): self.assertListEqual(_contents, _out) def test_scp_send_dir(self): + """ + Attempting to copy into a non-existent remote directory via scp_send() + without recurse=True should raise an SCPError. + """ test_file_data = 'test' local_filename = 'test_file' remote_test_dir, remote_filepath = 'remote_test_dir', 'test_file_copy' @@ -1277,10 +1281,33 @@ def test_scp_send_dir(self): remote_filename = os.path.sep.join([remote_test_dir, remote_filepath]) remote_file_abspath = os.path.expanduser('~/' + remote_filename) remote_test_dir_abspath = os.path.expanduser('~/' + remote_test_dir) + print(remote_file_abspath, remote_test_dir_abspath) try: cmds = self.client.scp_send(local_filename, remote_filename) joinall(cmds, raise_error=True) time.sleep(.2) + except Exception as ex: + self.assertIsInstance(ex, SCPError) + finally: + try: + os.unlink(local_filename) + shutil.rmtree(remote_test_dir_abspath) + except OSError: + pass + + def test_scp_send_dir_recurse(self): + test_file_data = 'test' + local_filename = 'test_file' + remote_test_dir, remote_filepath = 'remote_test_dir', 'test_file_copy' + with open(local_filename, 'w') as file_h: + file_h.writelines([test_file_data + os.linesep]) + remote_filename = os.path.sep.join([remote_test_dir, remote_filepath]) + remote_file_abspath = os.path.expanduser('~/' + remote_filename) + remote_test_dir_abspath = os.path.expanduser('~/' + remote_test_dir) + try: + cmds = self.client.scp_send(local_filename, remote_filename, recurse=True) + joinall(cmds, raise_error=True) + time.sleep(.2) self.assertTrue(os.path.isdir(remote_test_dir_abspath)) self.assertTrue(os.path.isfile(remote_file_abspath)) remote_file_data = open(remote_file_abspath, 'r').read() From 3c107616b99295b24c8759e67027ebe2127f3658 Mon Sep 17 00:00:00 2001 From: Casey Peel Date: Tue, 3 Dec 2019 12:30:02 -0800 Subject: [PATCH 3/7] Address flake8 issues --- pssh/clients/native/parallel.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pssh/clients/native/parallel.py b/pssh/clients/native/parallel.py index 9ff6151c..0d1ef265 100644 --- a/pssh/clients/native/parallel.py +++ b/pssh/clients/native/parallel.py @@ -236,7 +236,8 @@ def _run_command(self, host, command, sudo=False, user=None, logger.error("Failed to run on host %s - %s", host, ex) raise ex - def join(self, output, consume_output=False, timeout=None, encoding='utf-8'): + def join(self, output, consume_output=False, timeout=None, + encoding='utf-8'): """Wait until all remote commands in output have finished and retrieve exit codes. Does *not* block other commands from running in parallel. @@ -265,7 +266,8 @@ def join(self, output, consume_output=False, timeout=None, encoding='utf-8'): client = self.host_clients[host] channel = host_out.channel stdout, stderr = self.reset_output_generators( - host_out, client=client, channel=channel, timeout=timeout, encoding=encoding) + host_out, client=client, channel=channel, + timeout=timeout, encoding=encoding) try: client.wait_finished(channel, timeout=timeout) except Timeout: From bf2ac830ac1d4016e0154924808bf3c723cf5d0a Mon Sep 17 00:00:00 2001 From: Panos Date: Mon, 31 Aug 2020 17:27:32 +0100 Subject: [PATCH 4/7] Remove debug code --- tests/native/test_parallel_client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/native/test_parallel_client.py b/tests/native/test_parallel_client.py index bf20f7df..c99ac550 100644 --- a/tests/native/test_parallel_client.py +++ b/tests/native/test_parallel_client.py @@ -1307,16 +1307,17 @@ def test_scp_send_dir(self): remote_filename = os.path.sep.join([remote_test_dir, remote_filepath]) remote_file_abspath = os.path.expanduser('~/' + remote_filename) remote_test_dir_abspath = os.path.expanduser('~/' + remote_test_dir) - print(remote_file_abspath, remote_test_dir_abspath) try: cmds = self.client.scp_send(local_filename, remote_filename) joinall(cmds, raise_error=True) - time.sleep(.2) except Exception as ex: self.assertIsInstance(ex, SCPError) finally: try: os.unlink(local_filename) + except OSError: + pass + try: shutil.rmtree(remote_test_dir_abspath) except OSError: pass @@ -1333,7 +1334,6 @@ def test_scp_send_dir_recurse(self): try: cmds = self.client.scp_send(local_filename, remote_filename, recurse=True) joinall(cmds, raise_error=True) - time.sleep(.2) self.assertTrue(os.path.isdir(remote_test_dir_abspath)) self.assertTrue(os.path.isfile(remote_file_abspath)) remote_file_data = open(remote_file_abspath, 'r').read() From 109e5ffd6169ace05f12fb3e6a34e7c1304623d3 Mon Sep 17 00:00:00 2001 From: Panos Date: Mon, 31 Aug 2020 17:40:11 +0100 Subject: [PATCH 5/7] Cleanups --- pssh/clients/native/single.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pssh/clients/native/single.py b/pssh/clients/native/single.py index f7d5e1ac..c48fdd5d 100644 --- a/pssh/clients/native/single.py +++ b/pssh/clients/native/single.py @@ -470,7 +470,7 @@ def scp_recv(self, remote_file, local_file, recurse=False, sftp=None, encoding='utf-8'): """Copy remote file to local host via SCP. - Note - Remote directory listings are gather via SFTP when + Note - Remote directory listings are gathered via SFTP when ``recurse`` is enabled - SCP lacks directory list support. Enabling recursion therefore involves creating an extra SFTP channel and requires SFTP support on the server. @@ -491,8 +491,8 @@ def scp_recv(self, remote_file, local_file, recurse=False, sftp=None, :raises: :py:class:`IOError` on local file IO errors. :raises: :py:class:`OSError` on local OS errors like permission denied. """ - sftp = self._make_sftp() if (sftp is None and recurse) else sftp if recurse: + sftp = self._make_sftp() if sftp is None else sftp try: self._eagain(sftp.stat, remote_file) except (SFTPHandleError, SFTPProtocolError): From ad0f58002d3e487c8f3c66b4f608e69c36c4099a Mon Sep 17 00:00:00 2001 From: Panos Date: Mon, 31 Aug 2020 17:44:39 +0100 Subject: [PATCH 6/7] Updated changelog --- Changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/Changelog.rst b/Changelog.rst index 8fa31309..7cf48af3 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -9,6 +9,7 @@ Fixes * `ParallelSSHClient.copy_file` with recurse enabled and absolute destination path would create empty directory in home directory of user - #197. * `ParallelSSHClient.copy_file` and `scp_recv` with recurse enabled would not create remote directories when copying empty local directories. +* `ParallelSSHClient.scp_send` would require SFTP when recurse is off and remote destination path contains directory - #157. 1.12.1 ++++++ From b1ab4c397e18a6e608be3ec10a3b803d62aac474 Mon Sep 17 00:00:00 2001 From: Panos Date: Mon, 31 Aug 2020 17:48:48 +0100 Subject: [PATCH 7/7] Updated changelog --- Changelog.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index 7cf48af3..7d12294f 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -1,8 +1,8 @@ Change Log ============ -1.12.2 -++++++ +1.12.2 (unreleased) ++++++++++++++++++++ Fixes ------