diff --git a/Changelog.rst b/Changelog.rst index 7d12294f..9ef35b75 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -10,6 +10,7 @@ Fixes * `ParallelSSHClient.copy_file` with recurse enabled and absolute destination path would create empty directory in home directory of user - #197. * `ParallelSSHClient.copy_file` and `scp_recv` with recurse enabled would not create remote directories when copying empty local directories. * `ParallelSSHClient.scp_send` would require SFTP when recurse is off and remote destination path contains directory - #157. +* `ParallelSSHClient.scp_recv` could block infinitely on large - 200-300MB or more - files. 1.12.1 ++++++ diff --git a/pssh/clients/native/single.py b/pssh/clients/native/single.py index fdfd86f2..11f89643 100644 --- a/pssh/clients/native/single.py +++ b/pssh/clients/native/single.py @@ -524,15 +524,9 @@ def _scp_recv(self, remote_file, local_file): local_fh = open(local_file, 'wb') try: total = 0 - size, data = file_chan.read(size=fileinfo.st_size) - while size == LIBSSH2_ERROR_EAGAIN: - wait_select(self.session) - size, data = file_chan.read(size=fileinfo.st_size) - total += size - local_fh.write(data) while total < fileinfo.st_size: size, data = file_chan.read(size=fileinfo.st_size - total) - while size == LIBSSH2_ERROR_EAGAIN: + if size == LIBSSH2_ERROR_EAGAIN: wait_select(self.session) continue total += size diff --git a/tests/native/test_single_client.py b/tests/native/test_single_client.py index dcbf8084..eb333570 100644 --- a/tests/native/test_single_client.py +++ b/tests/native/test_single_client.py @@ -21,6 +21,7 @@ import time import subprocess import shutil +from hashlib import sha256 from gevent import socket, sleep, spawn @@ -353,3 +354,83 @@ def test_sftp_mkdir_rel_path(self): shutil.rmtree(_dir) except Exception: pass + + def test_scp_recv_large_file(self): + cur_dir = os.path.dirname(__file__) + file_name = 'file1' + file_copy_to = 'file_copied' + file_path_from = os.path.sep.join([cur_dir, file_name]) + file_copy_to_dirpath = os.path.expanduser('~/') + file_copy_to + for _path in (file_path_from, file_copy_to_dirpath): + try: + os.unlink(_path) + except OSError: + pass + try: + with open(file_path_from, 'wb') as fh: + # ~300MB + for _ in range(20000000): + fh.write(b"adsfasldkfjabafj") + self.client.scp_recv(file_path_from, file_copy_to_dirpath) + self.assertTrue(os.path.isfile(file_copy_to_dirpath)) + read_file_size = os.stat(file_path_from).st_size + written_file_size = os.stat(file_copy_to_dirpath).st_size + self.assertEqual(read_file_size, written_file_size) + sha = sha256() + with open(file_path_from, 'rb') as fh: + for block in fh: + sha.update(block) + read_file_hash = sha.hexdigest() + sha = sha256() + with open(file_copy_to_dirpath, 'rb') as fh: + for block in fh: + sha.update(block) + written_file_hash = sha.hexdigest() + self.assertEqual(read_file_hash, written_file_hash) + finally: + for _path in (file_path_from, file_copy_to_dirpath): + try: + os.unlink(_path) + except Exception: + pass + + def test_scp_send_large_file(self): + cur_dir = os.path.dirname(__file__) + file_name = 'file1' + file_copy_to = 'file_copied' + file_path_from = os.path.sep.join([cur_dir, file_name]) + file_copy_to_dirpath = os.path.expanduser('~/') + file_copy_to + for _path in (file_path_from, file_copy_to_dirpath): + try: + os.unlink(_path) + except OSError: + pass + try: + with open(file_path_from, 'wb') as fh: + # ~300MB + for _ in range(20000000): + fh.write(b"adsfasldkfjabafj") + self.client.scp_send(file_path_from, file_copy_to_dirpath) + self.assertTrue(os.path.isfile(file_copy_to_dirpath)) + # OS file flush race condition + sleep(.1) + read_file_size = os.stat(file_path_from).st_size + written_file_size = os.stat(file_copy_to_dirpath).st_size + self.assertEqual(read_file_size, written_file_size) + sha = sha256() + with open(file_path_from, 'rb') as fh: + for block in fh: + sha.update(block) + read_file_hash = sha.hexdigest() + sha = sha256() + with open(file_copy_to_dirpath, 'rb') as fh: + for block in fh: + sha.update(block) + written_file_hash = sha.hexdigest() + self.assertEqual(read_file_hash, written_file_hash) + finally: + for _path in (file_path_from, file_copy_to_dirpath): + try: + os.unlink(_path) + except Exception: + pass