From e4a04ca4828c65ae2333622e5f6aa118889d72ca Mon Sep 17 00:00:00 2001 From: Kincaid Savoie Date: Sat, 24 Oct 2015 16:04:23 -0600 Subject: [PATCH 01/24] copy_file now runs recursively when directory paths are supplied. --- pssh/ssh_client.py | 41 ++++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/pssh/ssh_client.py b/pssh/ssh_client.py index 71bf7236..c20085ac 100644 --- a/pssh/ssh_client.py +++ b/pssh/ssh_client.py @@ -25,6 +25,7 @@ import logging import paramiko import os +import itertools from socket import gaierror as sock_gaierror, error as sock_error from .exceptions import UnknownHostException, AuthenticationException, \ ConnectionErrorException, SSHException @@ -266,20 +267,30 @@ def copy_file(self, local_file, remote_file): :param remote_file: Remote filepath on remote host to copy file to :type remote_file: str """ - sftp = self._make_sftp() - destination = remote_file.split(os.path.sep) - remote_file = os.path.sep.join(destination) - destination = destination[:-1] - for directory in destination: + if os.path.isfile(local_file): + sftp = self._make_sftp() + destination = remote_file.split(os.path.sep) + remote_file = os.path.sep.join(destination) + destination = destination[:-1] + for directory in destination: + try: + sftp.stat(directory) + except IOError: + self.mkdir(sftp, directory) try: - sftp.stat(directory) - except IOError: - self.mkdir(sftp, directory) - try: - sftp.put(local_file, remote_file) - except Exception, error: - logger.error("Error occured copying file to host %s - %s", - self.host, error) + sftp.put(local_file, remote_file) + except Exception, error: + logger.error("Error occured copying file to host %s - %s", + self.host, error) + else: + logger.info("Copied local file %s to remote destination %s:%s", + local_file, self.host, remote_file) else: - logger.info("Copied local file %s to remote destination %s:%s", - local_file, self.host, remote_file) + file_list = os.listdir(local_file) + local_files = [] + remote_files = [] + for file_name in file_list: + local_files.append(os.path.join(local_file, file_name)) + remote_files.append(os.path.join(remote_file, file_name)) + for local_path, remote_path in itertools.izip(local_files, remote_files): + self.copy_file(local_path, remote_path) From 13eae16cb790776c36c961f3a8d3f99f3e8fabb6 Mon Sep 17 00:00:00 2001 From: Kincaid Savoie Date: Sat, 24 Oct 2015 16:05:06 -0600 Subject: [PATCH 02/24] Added test for recursive use of copy_file. --- tests/test_ssh_client.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/test_ssh_client.py b/tests/test_ssh_client.py index b6e7b96a..d05ccd8d 100644 --- a/tests/test_ssh_client.py +++ b/tests/test_ssh_client.py @@ -85,6 +85,28 @@ def test_ssh_client_sftp(self): os.rmdir(dirpath) del client + def test_ssh_client_directory(self): + """Tests copying directories with SSH client. Copy all the files from + local directory to server, then make sure they are all present.""" + test_file_data = 'test' + local_test_path = 'directory_test' + remote_test_path = 'directory_test_copied' + os.mkdir(local_test_path) + remote_file_paths = [] + for i in range(0, 10): + local_file_path = os.path.join(local_test_path, 'foo' + str(i)) + remote_file_path = os.path.join(remote_test_path, 'foo' + str(i)) + remote_file_paths.append(remote_file_path) + test_file = open(local_file_path, 'w') + test_file.write(test_file_data) + test_file.close() + client = SSHClient(self.host, port=self.listen_port, + pkey=self.user_key) + client.copy_file(local_test_path, remote_test_path) + for path in remote_file_paths: + self.assertTrue(os.path.isfile(path)) + + def test_ssh_agent_authentication(self): """Test authentication via SSH agent. Do not provide public key to use when creating SSHClient, From 700ffd574b03acc8295a5f9b809a044b6a0b32f7 Mon Sep 17 00:00:00 2001 From: Kincaid Savoie Date: Sat, 24 Oct 2015 16:09:22 -0600 Subject: [PATCH 03/24] Made code a bit more concise. --- pssh/ssh_client.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pssh/ssh_client.py b/pssh/ssh_client.py index c20085ac..2ada66a2 100644 --- a/pssh/ssh_client.py +++ b/pssh/ssh_client.py @@ -25,7 +25,6 @@ import logging import paramiko import os -import itertools from socket import gaierror as sock_gaierror, error as sock_error from .exceptions import UnknownHostException, AuthenticationException, \ ConnectionErrorException, SSHException @@ -287,10 +286,7 @@ def copy_file(self, local_file, remote_file): local_file, self.host, remote_file) else: file_list = os.listdir(local_file) - local_files = [] - remote_files = [] for file_name in file_list: - local_files.append(os.path.join(local_file, file_name)) - remote_files.append(os.path.join(remote_file, file_name)) - for local_path, remote_path in itertools.izip(local_files, remote_files): + local_path = os.path.join(local_file, file_name) + remote_path = os.path.join(remote_file, file_name) self.copy_file(local_path, remote_path) From 1d35cdb6e9a96f7941d7e830a86a05ca3a38989c Mon Sep 17 00:00:00 2001 From: Kincaid Savoie Date: Sat, 24 Oct 2015 16:20:36 -0600 Subject: [PATCH 04/24] Forgot to have test_ssh_client_directory clean up after itself. --- tests/test_ssh_client.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_ssh_client.py b/tests/test_ssh_client.py index d05ccd8d..bc5a2206 100644 --- a/tests/test_ssh_client.py +++ b/tests/test_ssh_client.py @@ -23,6 +23,7 @@ import gevent import socket import time +import shutil import unittest from pssh import SSHClient, ParallelSSHClient, UnknownHostException, AuthenticationException,\ logger, ConnectionErrorException, UnknownHostException, SSHException @@ -105,7 +106,8 @@ def test_ssh_client_directory(self): client.copy_file(local_test_path, remote_test_path) for path in remote_file_paths: self.assertTrue(os.path.isfile(path)) - + shutil.rmtree(local_test_path) + shutil.rmtree(remote_test_path) def test_ssh_agent_authentication(self): """Test authentication via SSH agent. From b3b2e9f54df8f7fa23c751d5851377c2d09e9a3a Mon Sep 17 00:00:00 2001 From: Kincaid Savoie Date: Sat, 24 Oct 2015 16:04:23 -0600 Subject: [PATCH 05/24] copy_file now runs recursively when directory paths are supplied. --- pssh/ssh_client.py | 42 +++++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/pssh/ssh_client.py b/pssh/ssh_client.py index fc6fd9a3..e8f33ef9 100644 --- a/pssh/ssh_client.py +++ b/pssh/ssh_client.py @@ -25,6 +25,7 @@ import logging import paramiko import os +import itertools from socket import gaierror as sock_gaierror, error as sock_error from .exceptions import UnknownHostException, AuthenticationException, \ ConnectionErrorException, SSHException @@ -274,21 +275,28 @@ def copy_file(self, local_file, remote_file): :param remote_file: Remote filepath on remote host to copy file to :type remote_file: str """ - sftp = self._make_sftp() - destination = [_dir for _dir in remote_file.split(os.path.sep) - if _dir][:-1] - if remote_file.startswith(os.path.sep): - destination[0] = os.path.sep + destination[0] - # import ipdb; ipdb.set_trace() - try: - sftp.stat(destination) - except IOError: - self.mkdir(sftp, destination) - try: - sftp.put(local_file, remote_file) - except Exception, error: - logger.error("Error occured copying file %s to remote destination %s:%s - %s", - local_file, self.host, remote_file, error) + if os.path.isfile(local_file): + sftp = self._make_sftp() + destination = [_dir for _dir in remote_file.split(os.path.sep) + if _dir][:-1] + if remote_file.startswith(os.path.sep): + destination[0] = os.path.sep + destination[0] + # import ipdb; ipdb.set_trace() + try: + sftp.stat(destination) + except IOError: + self.mkdir(sftp, destination) + try: + sftp.put(local_file, remote_file) + except Exception, error: + logger.error("Error occured copying file %s to remote destination %s:%s - %s", + local_file, self.host, remote_file, error) else: - logger.info("Copied local file %s to remote destination %s:%s", - local_file, self.host, remote_file) + file_list = os.listdir(local_file) + local_files = [] + remote_files = [] + for file_name in file_list: + local_files.append(os.path.join(local_file, file_name)) + remote_files.append(os.path.join(remote_file, file_name)) + for local_path, remote_path in itertools.izip(local_files, remote_files): + self.copy_file(local_path, remote_path) From 1c50939b4615de64a61032162bdfcdb4ec70182f Mon Sep 17 00:00:00 2001 From: Kincaid Savoie Date: Sat, 24 Oct 2015 16:05:06 -0600 Subject: [PATCH 06/24] Added test for recursive use of copy_file. --- tests/test_ssh_client.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/test_ssh_client.py b/tests/test_ssh_client.py index b6e7b96a..d05ccd8d 100644 --- a/tests/test_ssh_client.py +++ b/tests/test_ssh_client.py @@ -85,6 +85,28 @@ def test_ssh_client_sftp(self): os.rmdir(dirpath) del client + def test_ssh_client_directory(self): + """Tests copying directories with SSH client. Copy all the files from + local directory to server, then make sure they are all present.""" + test_file_data = 'test' + local_test_path = 'directory_test' + remote_test_path = 'directory_test_copied' + os.mkdir(local_test_path) + remote_file_paths = [] + for i in range(0, 10): + local_file_path = os.path.join(local_test_path, 'foo' + str(i)) + remote_file_path = os.path.join(remote_test_path, 'foo' + str(i)) + remote_file_paths.append(remote_file_path) + test_file = open(local_file_path, 'w') + test_file.write(test_file_data) + test_file.close() + client = SSHClient(self.host, port=self.listen_port, + pkey=self.user_key) + client.copy_file(local_test_path, remote_test_path) + for path in remote_file_paths: + self.assertTrue(os.path.isfile(path)) + + def test_ssh_agent_authentication(self): """Test authentication via SSH agent. Do not provide public key to use when creating SSHClient, From 9c4bb70fb1ccb7b4fc9dec8133552f61fa12b93d Mon Sep 17 00:00:00 2001 From: Kincaid Savoie Date: Sat, 24 Oct 2015 16:09:22 -0600 Subject: [PATCH 07/24] Made code a bit more concise. --- pssh/ssh_client.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pssh/ssh_client.py b/pssh/ssh_client.py index e8f33ef9..1d88f137 100644 --- a/pssh/ssh_client.py +++ b/pssh/ssh_client.py @@ -25,7 +25,6 @@ import logging import paramiko import os -import itertools from socket import gaierror as sock_gaierror, error as sock_error from .exceptions import UnknownHostException, AuthenticationException, \ ConnectionErrorException, SSHException @@ -293,10 +292,7 @@ def copy_file(self, local_file, remote_file): local_file, self.host, remote_file, error) else: file_list = os.listdir(local_file) - local_files = [] - remote_files = [] for file_name in file_list: - local_files.append(os.path.join(local_file, file_name)) - remote_files.append(os.path.join(remote_file, file_name)) - for local_path, remote_path in itertools.izip(local_files, remote_files): + local_path = os.path.join(local_file, file_name) + remote_path = os.path.join(remote_file, file_name) self.copy_file(local_path, remote_path) From 90a1b870af2973ca8aebf25de5480ca7e3561c1a Mon Sep 17 00:00:00 2001 From: Kincaid Savoie Date: Sat, 24 Oct 2015 16:20:36 -0600 Subject: [PATCH 08/24] Forgot to have test_ssh_client_directory clean up after itself. --- tests/test_ssh_client.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_ssh_client.py b/tests/test_ssh_client.py index d05ccd8d..bc5a2206 100644 --- a/tests/test_ssh_client.py +++ b/tests/test_ssh_client.py @@ -23,6 +23,7 @@ import gevent import socket import time +import shutil import unittest from pssh import SSHClient, ParallelSSHClient, UnknownHostException, AuthenticationException,\ logger, ConnectionErrorException, UnknownHostException, SSHException @@ -105,7 +106,8 @@ def test_ssh_client_directory(self): client.copy_file(local_test_path, remote_test_path) for path in remote_file_paths: self.assertTrue(os.path.isfile(path)) - + shutil.rmtree(local_test_path) + shutil.rmtree(remote_test_path) def test_ssh_agent_authentication(self): """Test authentication via SSH agent. From ca12de5ce1d7c971c92d399af45c153102afaa7b Mon Sep 17 00:00:00 2001 From: Kincaid Savoie Date: Tue, 27 Oct 2015 17:58:27 -0600 Subject: [PATCH 09/24] Broke directory copy logic into copy_dir; added recurse flag. --- pssh/ssh_client.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/pssh/ssh_client.py b/pssh/ssh_client.py index 1d88f137..013a4bac 100644 --- a/pssh/ssh_client.py +++ b/pssh/ssh_client.py @@ -263,7 +263,16 @@ def mkdir(self, sftp, directory): logger.error("Error occured creating directory %s on %s - %s", directory, self.host, error) - def copy_file(self, local_file, remote_file): + def _copy_dir(self, local_dir, remote_dir): + """Calls copy_file on every file in the specified directory, copying + them to the specified remote directory.""" + file_list = os.listdir(local_dir) + for file_name in file_list: + local_path = os.path.join(local_dir, file_name) + remote_path = os.path.join(remote_dir, file_name) + self.copy_file(local_path, remote_path) + + def copy_file(self, local_file, remote_file, recurse=False): """Copy local file to host via SFTP/SCP Copy is done natively using SFTP/SCP version 2 protocol, no scp command \ @@ -274,7 +283,7 @@ def copy_file(self, local_file, remote_file): :param remote_file: Remote filepath on remote host to copy file to :type remote_file: str """ - if os.path.isfile(local_file): + if os.path.isfile(local_file) or not recurse: sftp = self._make_sftp() destination = [_dir for _dir in remote_file.split(os.path.sep) if _dir][:-1] @@ -282,17 +291,13 @@ def copy_file(self, local_file, remote_file): destination[0] = os.path.sep + destination[0] # import ipdb; ipdb.set_trace() try: - sftp.stat(destination) + sftp.stat(destination[0]) except IOError: - self.mkdir(sftp, destination) + self.mkdir(sftp, destination[0]) try: sftp.put(local_file, remote_file) except Exception, error: logger.error("Error occured copying file %s to remote destination %s:%s - %s", local_file, self.host, remote_file, error) else: - file_list = os.listdir(local_file) - for file_name in file_list: - local_path = os.path.join(local_file, file_name) - remote_path = os.path.join(remote_file, file_name) - self.copy_file(local_path, remote_path) + self._copy_dir(local_file, remote_file) From a3ae18586527a078265008c6504f744c76e6e070 Mon Sep 17 00:00:00 2001 From: Kincaid Savoie Date: Tue, 27 Oct 2015 18:15:58 -0600 Subject: [PATCH 10/24] Updated test_ssh_client_directory to use recurse flag. --- tests/test_ssh_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_ssh_client.py b/tests/test_ssh_client.py index bc5a2206..1d90b59c 100644 --- a/tests/test_ssh_client.py +++ b/tests/test_ssh_client.py @@ -103,7 +103,7 @@ def test_ssh_client_directory(self): test_file.close() client = SSHClient(self.host, port=self.listen_port, pkey=self.user_key) - client.copy_file(local_test_path, remote_test_path) + client.copy_file(local_test_path, remote_test_path, recurse=True) for path in remote_file_paths: self.assertTrue(os.path.isfile(path)) shutil.rmtree(local_test_path) From 9faef2b74af72856100b36c876f5a7660f56b4c8 Mon Sep 17 00:00:00 2001 From: Kincaid Savoie Date: Wed, 28 Oct 2015 10:46:23 -0600 Subject: [PATCH 11/24] Made copy_file branching simpler; updated copy_file documentation. --- pssh/ssh_client.py | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/pssh/ssh_client.py b/pssh/ssh_client.py index 013a4bac..89fbbd49 100644 --- a/pssh/ssh_client.py +++ b/pssh/ssh_client.py @@ -282,22 +282,23 @@ def copy_file(self, local_file, remote_file, recurse=False): :type local_file: str :param remote_file: Remote filepath on remote host to copy file to :type remote_file: str + :param recurse: Whether or not to descend into directories recursively. + :type recurse: bool """ - if os.path.isfile(local_file) or not recurse: - sftp = self._make_sftp() - destination = [_dir for _dir in remote_file.split(os.path.sep) - if _dir][:-1] - if remote_file.startswith(os.path.sep): - destination[0] = os.path.sep + destination[0] - # import ipdb; ipdb.set_trace() - try: - sftp.stat(destination[0]) - except IOError: - self.mkdir(sftp, destination[0]) - try: - sftp.put(local_file, remote_file) - except Exception, error: - logger.error("Error occured copying file %s to remote destination %s:%s - %s", - local_file, self.host, remote_file, error) - else: - self._copy_dir(local_file, remote_file) + if os.path.isdir(local_file) and recurse: + return self._copy_dir(local_file, remote_file) + sftp = self._make_sftp() + destination = [_dir for _dir in remote_file.split(os.path.sep) + if _dir][:-1] + if remote_file.startswith(os.path.sep): + destination[0] = os.path.sep + destination[0] + # import ipdb; ipdb.set_trace() + try: + sftp.stat(destination[0]) + except IOError: + self.mkdir(sftp, destination[0]) + try: + sftp.put(local_file, remote_file) + except Exception, error: + logger.error("Error occured copying file %s to remote destination %s:%s - %s", + local_file, self.host, remote_file, error) From 2379b7a684af3db01a0c8e9fb294c06367d85ea3 Mon Sep 17 00:00:00 2001 From: Kincaid Savoie Date: Sat, 24 Oct 2015 16:04:23 -0600 Subject: [PATCH 12/24] copy_file now runs recursively when directory paths are supplied. --- pssh/ssh_client.py | 42 +++++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/pssh/ssh_client.py b/pssh/ssh_client.py index 3c22e338..640f6868 100644 --- a/pssh/ssh_client.py +++ b/pssh/ssh_client.py @@ -25,6 +25,7 @@ import logging import paramiko import os +import itertools from socket import gaierror as sock_gaierror, error as sock_error from .exceptions import UnknownHostException, AuthenticationException, \ ConnectionErrorException, SSHException @@ -296,21 +297,28 @@ def copy_file(self, local_file, remote_file): :param remote_file: Remote filepath on remote host to copy file to :type remote_file: str """ - sftp = self._make_sftp() - destination = [_dir for _dir in remote_file.split(os.path.sep) - if _dir][:-1][0] - if remote_file.startswith(os.path.sep): - destination = os.path.sep + destination - try: - sftp.stat(destination) - except IOError: - self.mkdir(sftp, destination) - sftp.chdir() - try: - sftp.put(local_file, remote_file) - except Exception, error: - logger.error("Error occured copying file %s to remote destination %s:%s - %s", - local_file, self.host, remote_file, error) + if os.path.isfile(local_file): + sftp = self._make_sftp() + destination = [_dir for _dir in remote_file.split(os.path.sep) + if _dir][:-1][0] + if remote_file.startswith(os.path.sep): + destination = os.path.sep + destination + try: + sftp.stat(destination) + except IOError: + self.mkdir(sftp, destination) + sftp.chdir() + try: + sftp.put(local_file, remote_file) + except Exception, error: + logger.error("Error occured copying file %s to remote destination %s:%s - %s", + local_file, self.host, remote_file, error) else: - logger.info("Copied local file %s to remote destination %s:%s", - local_file, self.host, remote_file) + file_list = os.listdir(local_file) + local_files = [] + remote_files = [] + for file_name in file_list: + local_files.append(os.path.join(local_file, file_name)) + remote_files.append(os.path.join(remote_file, file_name)) + for local_path, remote_path in itertools.izip(local_files, remote_files): + self.copy_file(local_path, remote_path) From 363e53dba72d613fc168ed32997eacc365c12aa6 Mon Sep 17 00:00:00 2001 From: Kincaid Savoie Date: Sat, 24 Oct 2015 16:05:06 -0600 Subject: [PATCH 13/24] Added test for recursive use of copy_file. --- tests/test_ssh_client.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/test_ssh_client.py b/tests/test_ssh_client.py index 6f5e5977..81505936 100644 --- a/tests/test_ssh_client.py +++ b/tests/test_ssh_client.py @@ -141,6 +141,28 @@ def test_ssh_client_sftp(self): os.rmdir(dirpath) del client + def test_ssh_client_directory(self): + """Tests copying directories with SSH client. Copy all the files from + local directory to server, then make sure they are all present.""" + test_file_data = 'test' + local_test_path = 'directory_test' + remote_test_path = 'directory_test_copied' + os.mkdir(local_test_path) + remote_file_paths = [] + for i in range(0, 10): + local_file_path = os.path.join(local_test_path, 'foo' + str(i)) + remote_file_path = os.path.join(remote_test_path, 'foo' + str(i)) + remote_file_paths.append(remote_file_path) + test_file = open(local_file_path, 'w') + test_file.write(test_file_data) + test_file.close() + client = SSHClient(self.host, port=self.listen_port, + pkey=self.user_key) + client.copy_file(local_test_path, remote_test_path) + for path in remote_file_paths: + self.assertTrue(os.path.isfile(path)) + + def test_ssh_agent_authentication(self): """Test authentication via SSH agent. Do not provide public key to use when creating SSHClient, From 6d0022e6624729bc5cad22a7c379184068f28647 Mon Sep 17 00:00:00 2001 From: Kincaid Savoie Date: Sat, 24 Oct 2015 16:09:22 -0600 Subject: [PATCH 14/24] Made code a bit more concise. --- pssh/ssh_client.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pssh/ssh_client.py b/pssh/ssh_client.py index 640f6868..81836b88 100644 --- a/pssh/ssh_client.py +++ b/pssh/ssh_client.py @@ -25,7 +25,6 @@ import logging import paramiko import os -import itertools from socket import gaierror as sock_gaierror, error as sock_error from .exceptions import UnknownHostException, AuthenticationException, \ ConnectionErrorException, SSHException @@ -315,10 +314,7 @@ def copy_file(self, local_file, remote_file): local_file, self.host, remote_file, error) else: file_list = os.listdir(local_file) - local_files = [] - remote_files = [] for file_name in file_list: - local_files.append(os.path.join(local_file, file_name)) - remote_files.append(os.path.join(remote_file, file_name)) - for local_path, remote_path in itertools.izip(local_files, remote_files): + local_path = os.path.join(local_file, file_name) + remote_path = os.path.join(remote_file, file_name) self.copy_file(local_path, remote_path) From 4b38c149008643304aeaf54ad028e306b31ec0a8 Mon Sep 17 00:00:00 2001 From: Kincaid Savoie Date: Sat, 24 Oct 2015 16:20:36 -0600 Subject: [PATCH 15/24] Forgot to have test_ssh_client_directory clean up after itself. --- tests/test_ssh_client.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_ssh_client.py b/tests/test_ssh_client.py index 81505936..291bf1c1 100644 --- a/tests/test_ssh_client.py +++ b/tests/test_ssh_client.py @@ -23,6 +23,7 @@ import gevent import socket import time +import shutil import unittest from pssh import SSHClient, ParallelSSHClient, UnknownHostException, AuthenticationException,\ logger, ConnectionErrorException, UnknownHostException, SSHException @@ -161,7 +162,8 @@ def test_ssh_client_directory(self): client.copy_file(local_test_path, remote_test_path) for path in remote_file_paths: self.assertTrue(os.path.isfile(path)) - + shutil.rmtree(local_test_path) + shutil.rmtree(remote_test_path) def test_ssh_agent_authentication(self): """Test authentication via SSH agent. From 655fba7fa7067a00f9c801781a9196062e758556 Mon Sep 17 00:00:00 2001 From: Kincaid Savoie Date: Sat, 24 Oct 2015 16:04:23 -0600 Subject: [PATCH 16/24] copy_file now runs recursively when directory paths are supplied. --- pssh/ssh_client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pssh/ssh_client.py b/pssh/ssh_client.py index 81836b88..33b544d3 100644 --- a/pssh/ssh_client.py +++ b/pssh/ssh_client.py @@ -25,6 +25,7 @@ import logging import paramiko import os +import itertools from socket import gaierror as sock_gaierror, error as sock_error from .exceptions import UnknownHostException, AuthenticationException, \ ConnectionErrorException, SSHException From 6e09d52215bee372ff6ced7834e107d03bd5abb3 Mon Sep 17 00:00:00 2001 From: Kincaid Savoie Date: Sat, 24 Oct 2015 16:09:22 -0600 Subject: [PATCH 17/24] Made code a bit more concise. --- pssh/ssh_client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pssh/ssh_client.py b/pssh/ssh_client.py index 33b544d3..81836b88 100644 --- a/pssh/ssh_client.py +++ b/pssh/ssh_client.py @@ -25,7 +25,6 @@ import logging import paramiko import os -import itertools from socket import gaierror as sock_gaierror, error as sock_error from .exceptions import UnknownHostException, AuthenticationException, \ ConnectionErrorException, SSHException From 4af7aa0fe318a75dbb9255ed9b04dec882a26eed Mon Sep 17 00:00:00 2001 From: Kincaid Savoie Date: Tue, 27 Oct 2015 17:58:27 -0600 Subject: [PATCH 18/24] Broke directory copy logic into copy_dir; added recurse flag. --- pssh/ssh_client.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/pssh/ssh_client.py b/pssh/ssh_client.py index 81836b88..859ce79c 100644 --- a/pssh/ssh_client.py +++ b/pssh/ssh_client.py @@ -285,7 +285,16 @@ def mkdir(self, sftp, directory): return self.mkdir(sftp, sub_dirs) return True - def copy_file(self, local_file, remote_file): + def _copy_dir(self, local_dir, remote_dir): + """Calls copy_file on every file in the specified directory, copying + them to the specified remote directory.""" + file_list = os.listdir(local_dir) + for file_name in file_list: + local_path = os.path.join(local_dir, file_name) + remote_path = os.path.join(remote_dir, file_name) + self.copy_file(local_path, remote_path) + + def copy_file(self, local_file, remote_file, recurse=False): """Copy local file to host via SFTP/SCP Copy is done natively using SFTP/SCP version 2 protocol, no scp command \ @@ -296,14 +305,14 @@ def copy_file(self, local_file, remote_file): :param remote_file: Remote filepath on remote host to copy file to :type remote_file: str """ - if os.path.isfile(local_file): + if os.path.isfile(local_file) or not recurse: sftp = self._make_sftp() destination = [_dir for _dir in remote_file.split(os.path.sep) if _dir][:-1][0] if remote_file.startswith(os.path.sep): destination = os.path.sep + destination try: - sftp.stat(destination) + sftp.stat(destination[0]) except IOError: self.mkdir(sftp, destination) sftp.chdir() @@ -313,8 +322,4 @@ def copy_file(self, local_file, remote_file): logger.error("Error occured copying file %s to remote destination %s:%s - %s", local_file, self.host, remote_file, error) else: - file_list = os.listdir(local_file) - for file_name in file_list: - local_path = os.path.join(local_file, file_name) - remote_path = os.path.join(remote_file, file_name) - self.copy_file(local_path, remote_path) + self._copy_dir(local_file, remote_file) From ce4ae41caa6d627b26966e98fc63bda225dc9d4c Mon Sep 17 00:00:00 2001 From: Kincaid Savoie Date: Tue, 27 Oct 2015 18:15:58 -0600 Subject: [PATCH 19/24] Updated test_ssh_client_directory to use recurse flag. --- tests/test_ssh_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_ssh_client.py b/tests/test_ssh_client.py index 291bf1c1..7ebc8ae9 100644 --- a/tests/test_ssh_client.py +++ b/tests/test_ssh_client.py @@ -159,7 +159,7 @@ def test_ssh_client_directory(self): test_file.close() client = SSHClient(self.host, port=self.listen_port, pkey=self.user_key) - client.copy_file(local_test_path, remote_test_path) + client.copy_file(local_test_path, remote_test_path, recurse=True) for path in remote_file_paths: self.assertTrue(os.path.isfile(path)) shutil.rmtree(local_test_path) From cac9ba2a1efd4d11b75973c7627fc847f4d0b34c Mon Sep 17 00:00:00 2001 From: Kincaid Savoie Date: Wed, 28 Oct 2015 10:46:23 -0600 Subject: [PATCH 20/24] Made copy_file branching simpler; updated copy_file documentation. --- pssh/ssh_client.py | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/pssh/ssh_client.py b/pssh/ssh_client.py index 859ce79c..1ad0231a 100644 --- a/pssh/ssh_client.py +++ b/pssh/ssh_client.py @@ -304,22 +304,23 @@ def copy_file(self, local_file, remote_file, recurse=False): :type local_file: str :param remote_file: Remote filepath on remote host to copy file to :type remote_file: str + :param recurse: Whether or not to descend into directories recursively. + :type recurse: bool """ - if os.path.isfile(local_file) or not recurse: - sftp = self._make_sftp() - destination = [_dir for _dir in remote_file.split(os.path.sep) - if _dir][:-1][0] - if remote_file.startswith(os.path.sep): - destination = os.path.sep + destination - try: - sftp.stat(destination[0]) - except IOError: - self.mkdir(sftp, destination) - sftp.chdir() - try: - sftp.put(local_file, remote_file) - except Exception, error: - logger.error("Error occured copying file %s to remote destination %s:%s - %s", - local_file, self.host, remote_file, error) - else: - self._copy_dir(local_file, remote_file) + if os.path.isdir(local_file) and recurse: + return self._copy_dir(local_file, remote_file) + sftp = self._make_sftp() + destination = [_dir for _dir in remote_file.split(os.path.sep) + if _dir][:-1] + if remote_file.startswith(os.path.sep): + destination[0] = os.path.sep + destination[0] + # import ipdb; ipdb.set_trace() + try: + sftp.stat(destination[0]) + except IOError: + self.mkdir(sftp, destination[0]) + try: + sftp.put(local_file, remote_file) + except Exception, error: + logger.error("Error occured copying file %s to remote destination %s:%s - %s", + local_file, self.host, remote_file, error) From 6ff45eef2120f6cbf3440f80489825dbeb642a09 Mon Sep 17 00:00:00 2001 From: Kincaid Savoie Date: Wed, 28 Oct 2015 19:01:38 -0600 Subject: [PATCH 21/24] Fixed messy merge. --- pssh/ssh_client.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/pssh/ssh_client.py b/pssh/ssh_client.py index 1ad0231a..22beb8b1 100644 --- a/pssh/ssh_client.py +++ b/pssh/ssh_client.py @@ -311,16 +311,19 @@ def copy_file(self, local_file, remote_file, recurse=False): return self._copy_dir(local_file, remote_file) sftp = self._make_sftp() destination = [_dir for _dir in remote_file.split(os.path.sep) - if _dir][:-1] + if _dir][:-1][0] if remote_file.startswith(os.path.sep): - destination[0] = os.path.sep + destination[0] - # import ipdb; ipdb.set_trace() + destination = os.path.sep + destination try: - sftp.stat(destination[0]) + sftp.stat(destination) except IOError: - self.mkdir(sftp, destination[0]) + self.mkdir(sftp, destination) + sftp.chdir() try: sftp.put(local_file, remote_file) except Exception, error: logger.error("Error occured copying file %s to remote destination %s:%s - %s", local_file, self.host, remote_file, error) + else: + logger.info("Copied local file %s to remote destination %s:%s", + local_file, self.host, remote_file) From a890f4158d75f16417c94fdb45af65b3d758386f Mon Sep 17 00:00:00 2001 From: Kincaid Savoie Date: Wed, 28 Oct 2015 21:42:14 -0600 Subject: [PATCH 22/24] Forgot to enable recurse when recursively calling copy_file. --- pssh/ssh_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pssh/ssh_client.py b/pssh/ssh_client.py index 22beb8b1..334e31f3 100644 --- a/pssh/ssh_client.py +++ b/pssh/ssh_client.py @@ -292,7 +292,7 @@ def _copy_dir(self, local_dir, remote_dir): for file_name in file_list: local_path = os.path.join(local_dir, file_name) remote_path = os.path.join(remote_dir, file_name) - self.copy_file(local_path, remote_path) + self.copy_file(local_path, remote_path, recurse=True) def copy_file(self, local_file, remote_file, recurse=False): """Copy local file to host via SFTP/SCP From d14f7198ea378dbcdb84ac79c4d77e9c3e16292f Mon Sep 17 00:00:00 2001 From: Kincaid Savoie Date: Fri, 30 Oct 2015 10:17:27 -0600 Subject: [PATCH 23/24] ssh_client now throws an exception when given a directory and recurse is not set --- pssh/ssh_client.py | 3 +++ tests/test_ssh_client.py | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/pssh/ssh_client.py b/pssh/ssh_client.py index 334e31f3..9d5fdd66 100644 --- a/pssh/ssh_client.py +++ b/pssh/ssh_client.py @@ -309,6 +309,9 @@ def copy_file(self, local_file, remote_file, recurse=False): """ if os.path.isdir(local_file) and recurse: return self._copy_dir(local_file, remote_file) + elif os.path.isdir(local_file) and not recurse: + raise ValueError("Recurse must be true if local_file is a " + "directory.") sftp = self._make_sftp() destination = [_dir for _dir in remote_file.split(os.path.sep) if _dir][:-1][0] diff --git a/tests/test_ssh_client.py b/tests/test_ssh_client.py index 7ebc8ae9..7db46b87 100644 --- a/tests/test_ssh_client.py +++ b/tests/test_ssh_client.py @@ -165,6 +165,26 @@ def test_ssh_client_directory(self): shutil.rmtree(local_test_path) shutil.rmtree(remote_test_path) + def test_ssh_client_directory_no_recurse(self): + """Tests copying directories with SSH client. Copy all the files from + local directory to server, then make sure they are all present.""" + test_file_data = 'test' + local_test_path = 'directory_test' + remote_test_path = 'directory_test_copied' + os.mkdir(local_test_path) + remote_file_paths = [] + for i in range(0, 10): + local_file_path = os.path.join(local_test_path, 'foo' + str(i)) + remote_file_path = os.path.join(remote_test_path, 'foo' + str(i)) + remote_file_paths.append(remote_file_path) + test_file = open(local_file_path, 'w') + test_file.write(test_file_data) + test_file.close() + client = SSHClient(self.host, port=self.listen_port, + pkey=self.user_key) + self.assertRaises(ValueError, client.copy_file, local_test_path, remote_test_path) + shutil.rmtree(local_test_path) + def test_ssh_agent_authentication(self): """Test authentication via SSH agent. Do not provide public key to use when creating SSHClient, From 8df29c98848b302af05e4eb64127c5df126ef83b Mon Sep 17 00:00:00 2001 From: Kincaid Savoie Date: Tue, 3 Nov 2015 19:24:28 -0700 Subject: [PATCH 24/24] update docstring comments for copy_file and _copy_dir --- pssh/ssh_client.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pssh/ssh_client.py b/pssh/ssh_client.py index 9d5fdd66..ad271ffe 100644 --- a/pssh/ssh_client.py +++ b/pssh/ssh_client.py @@ -286,7 +286,7 @@ def mkdir(self, sftp, directory): return True def _copy_dir(self, local_dir, remote_dir): - """Calls copy_file on every file in the specified directory, copying + """Call copy_file on every file in the specified directory, copying them to the specified remote directory.""" file_list = os.listdir(local_dir) for file_name in file_list: @@ -306,6 +306,9 @@ def copy_file(self, local_file, remote_file, recurse=False): :type remote_file: str :param recurse: Whether or not to descend into directories recursively. :type recurse: bool + + :raises: :mod:'ValueError' when a directory is supplied to local_file \ + and recurse is not set """ if os.path.isdir(local_file) and recurse: return self._copy_dir(local_file, remote_file)