Eagain inf loop

Changelog.rst

@@ -10,6 +10,7 @@ Fixes
 * `ParallelSSHClient.copy_file` with recurse enabled and absolute destination path would create empty directory in home directory of user - #197.
 * `ParallelSSHClient.copy_file` and `scp_recv` with recurse enabled would not create remote directories when copying empty local directories.
 * `ParallelSSHClient.scp_send` would require SFTP when recurse is off and remote destination path contains directory - #157.
+* `ParallelSSHClient.scp_recv` could block infinitely on large - 200-300MB or more - files.
 
 1.12.1
 ++++++

pssh/clients/native/single.py

@@ -524,15 +524,9 @@ def _scp_recv(self, remote_file, local_file):
         local_fh = open(local_file, 'wb')
         try:
             total = 0
-            size, data = file_chan.read(size=fileinfo.st_size)
-            while size == LIBSSH2_ERROR_EAGAIN:
-                wait_select(self.session)
-                size, data = file_chan.read(size=fileinfo.st_size)
-            total += size
-            local_fh.write(data)
             while total < fileinfo.st_size:
                 size, data = file_chan.read(size=fileinfo.st_size - total)
-                while size == LIBSSH2_ERROR_EAGAIN:
+                if size == LIBSSH2_ERROR_EAGAIN:
                     wait_select(self.session)
                     continue
                 total += size

tests/native/test_single_client.py

@@ -21,6 +21,7 @@
 import time
 import subprocess
 import shutil
+from hashlib import sha256
 
 from gevent import socket, sleep, spawn
 
@@ -353,3 +354,83 @@ def test_sftp_mkdir_rel_path(self):
                     shutil.rmtree(_dir)
                 except Exception:
                     pass
+
+    def test_scp_recv_large_file(self):
+        cur_dir = os.path.dirname(__file__)
+        file_name = 'file1'
+        file_copy_to = 'file_copied'
+        file_path_from = os.path.sep.join([cur_dir, file_name])
+        file_copy_to_dirpath = os.path.expanduser('~/') + file_copy_to
+        for _path in (file_path_from, file_copy_to_dirpath):
+            try:
+                os.unlink(_path)
+            except OSError:
+                pass
+        try:
+            with open(file_path_from, 'wb') as fh:
+                # ~300MB
+                for _ in range(20000000):
+                    fh.write(b"adsfasldkfjabafj")
+            self.client.scp_recv(file_path_from, file_copy_to_dirpath)
+            self.assertTrue(os.path.isfile(file_copy_to_dirpath))
+            read_file_size = os.stat(file_path_from).st_size
+            written_file_size = os.stat(file_copy_to_dirpath).st_size
+            self.assertEqual(read_file_size, written_file_size)
+            sha = sha256()
+            with open(file_path_from, 'rb') as fh:
+                for block in fh:
+                    sha.update(block)
+            read_file_hash = sha.hexdigest()
+            sha = sha256()
+            with open(file_copy_to_dirpath, 'rb') as fh:
+                for block in fh:
+                    sha.update(block)
+            written_file_hash = sha.hexdigest()
+            self.assertEqual(read_file_hash, written_file_hash)
+        finally:
+            for _path in (file_path_from, file_copy_to_dirpath):
+                try:
+                    os.unlink(_path)
+                except Exception:
+                    pass
+
+    def test_scp_send_large_file(self):
+        cur_dir = os.path.dirname(__file__)
+        file_name = 'file1'
+        file_copy_to = 'file_copied'
+        file_path_from = os.path.sep.join([cur_dir, file_name])
+        file_copy_to_dirpath = os.path.expanduser('~/') + file_copy_to
+        for _path in (file_path_from, file_copy_to_dirpath):
+            try:
+                os.unlink(_path)
+            except OSError:
+                pass
+        try:
+            with open(file_path_from, 'wb') as fh:
+                # ~300MB
+                for _ in range(20000000):
+                    fh.write(b"adsfasldkfjabafj")
+            self.client.scp_send(file_path_from, file_copy_to_dirpath)
+            self.assertTrue(os.path.isfile(file_copy_to_dirpath))
+            # OS file flush race condition
+            sleep(.1)
+            read_file_size = os.stat(file_path_from).st_size
+            written_file_size = os.stat(file_copy_to_dirpath).st_size
+            self.assertEqual(read_file_size, written_file_size)
+            sha = sha256()
+            with open(file_path_from, 'rb') as fh:
+                for block in fh:
+                    sha.update(block)
+            read_file_hash = sha.hexdigest()
+            sha = sha256()
+            with open(file_copy_to_dirpath, 'rb') as fh:
+                for block in fh:
+                    sha.update(block)
+            written_file_hash = sha.hexdigest()
+            self.assertEqual(read_file_hash, written_file_hash)
+        finally:
+            for _path in (file_path_from, file_copy_to_dirpath):
+                try:
+                    os.unlink(_path)
+                except Exception:
+                    pass