From b03b4181a6a9603b9dde4fff4ee4cc5464dfe6ed Mon Sep 17 00:00:00 2001
From: Andrew Shulgin <andrewshulginua@gmail.com>
Date: Tue, 26 Mar 2019 20:12:45 +0200
Subject: [PATCH] Fixed FileProducer._data_wrapper - do not replace LF with
 CRLF if CRLF already

---
 pyftpdlib/handlers.py             | 23 +++++++++++++-
 pyftpdlib/test/__init__.py        |  5 +++-
 pyftpdlib/test/test_functional.py | 50 ++++++++++++++++++++-----------
 3 files changed, 58 insertions(+), 20 deletions(-)

diff --git a/pyftpdlib/handlers.py b/pyftpdlib/handlers.py
index 5851c1f7..b651a103 100644
--- a/pyftpdlib/handlers.py
+++ b/pyftpdlib/handlers.py
@@ -1028,11 +1028,32 @@ def __init__(self, file, type):
         """
         self.file = file
         self.type = type
+        self._prev_chunk_endswith_cr = False
         if type == 'a' and os.linesep != '\r\n':
-            self._data_wrapper = lambda x: x.replace(b(os.linesep), b'\r\n')
+            self._data_wrapper = self._posix_ascii_data_wrapper
         else:
             self._data_wrapper = None
 
+    def _posix_ascii_data_wrapper(self, chunk):
+        """The data wrapper used for sending data in ASCII mode on
+        systems using a single line terminator, handling those cases
+        where CRLF ('\r\n') gets delivered in two chunks.
+        """
+        chunk = bytearray(chunk)
+        pos = 0
+        if self._prev_chunk_endswith_cr and chunk.startswith(b'\n'):
+            pos += 1
+        while True:
+            pos = chunk.find(b'\n', pos)
+            if pos == -1:
+                break
+            if chunk[pos - 1] != 13:
+                chunk.insert(pos, 13)
+                pos += 1
+            pos += 1
+        self._prev_chunk_endswith_cr = chunk.endswith(b'\r')
+        return chunk
+
     def more(self):
         """Attempt a chunk of data of size self.buffer_size."""
         try:
diff --git a/pyftpdlib/test/__init__.py b/pyftpdlib/test/__init__.py
index 3615ee6a..fd182b1e 100644
--- a/pyftpdlib/test/__init__.py
+++ b/pyftpdlib/test/__init__.py
@@ -37,7 +37,10 @@
     unittest.TestCase.assertRaisesRegex = unittest.TestCase.assertRaisesRegexp
 
 if os.name == 'posix':
-    import sendfile
+    try:
+        import sendfile
+    except ImportError:
+        sendfile = None
 else:
     sendfile = None
 
diff --git a/pyftpdlib/test/test_functional.py b/pyftpdlib/test/test_functional.py
index 59659a99..ce09af4e 100644
--- a/pyftpdlib/test/test_functional.py
+++ b/pyftpdlib/test/test_functional.py
@@ -67,7 +67,10 @@
 import ssl
 
 if POSIX:
-    import sendfile
+    try:
+        import sendfile
+    except ImportError:
+        sendfile = None
 else:
     sendfile = None
 
@@ -961,12 +964,24 @@ class TestFtpStoreDataNoSendfile(TestFtpStoreData):
 
 
 class TestFtpRetrieveData(unittest.TestCase):
-
-    "Test RETR, REST, TYPE"
+    """Test RETR, REST, TYPE"""
     server_class = MProcessTestFTPd
     client_class = ftplib.FTP
     use_sendfile = None
 
+    def retrieve_ascii(self, cmd, callback, blocksize=8192, rest=None):
+        """Like retrbinary but uses TYPE A instead."""
+        self.client.voidcmd('type a')
+        with contextlib.closing(
+                self.client.transfercmd(cmd, rest)) as conn:
+            conn.settimeout(TIMEOUT)
+            while True:
+                data = conn.recv(blocksize)
+                if not data:
+                    break
+                callback(data)
+        return self.client.voidresp()
+
     def setUp(self):
         self.server = self.server_class()
         if self.use_sendfile is not None:
@@ -1006,31 +1021,30 @@ def test_retr(self):
                           "retr " + bogus, lambda x: x)
 
     def test_retr_ascii(self):
-        # Test RETR in ASCII mode.
-
-        def retrieve(cmd, callback, blocksize=8192, rest=None):
-            # like retrbinary but uses TYPE A instead
-            self.client.voidcmd('type a')
-            with contextlib.closing(
-                    self.client.transfercmd(cmd, rest)) as conn:
-                conn.settimeout(TIMEOUT)
-                while True:
-                    data = conn.recv(blocksize)
-                    if not data:
-                        break
-                    callback(data)
-            return self.client.voidresp()
+        """Test RETR in ASCII mode."""
 
         data = (b'abcde12345' + b(os.linesep)) * 100000
         self.file.write(data)
         self.file.close()
-        retrieve("retr " + TESTFN, self.dummyfile.write)
+        self.retrieve_ascii("retr " + TESTFN, self.dummyfile.write)
         expected = data.replace(b(os.linesep), b'\r\n')
         self.dummyfile.seek(0)
         datafile = self.dummyfile.read()
         self.assertEqual(len(expected), len(datafile))
         self.assertEqual(hash(expected), hash(datafile))
 
+    def test_retr_ascii_already_crlf(self):
+        """Test ASCII mode RETR for data with CRLF line endings."""
+
+        data = b'abcde12345\r\n' * 100000
+        self.file.write(data)
+        self.file.close()
+        self.retrieve_ascii("retr " + TESTFN, self.dummyfile.write)
+        self.dummyfile.seek(0)
+        datafile = self.dummyfile.read()
+        self.assertEqual(len(data), len(datafile))
+        self.assertEqual(hash(data), hash(datafile))
+
     @retry_on_failure()
     def test_restore_on_retr(self):
         data = b'abcde12345' * 1000000