Merge e362b2e into 3514646

HISTORY.rst

@@ -5,6 +5,7 @@ Version: 1.5.5 - XXXX-XX-XX
 
 **Enhancements**
 
+- #494: Implemented CR to CRLF line ending conversion for ASCII downloads.
 - #495: colored test output.
 
 **Bug fixes**

pyftpdlib/handlers.py

@@ -55,6 +55,7 @@
 from .log import logger
 
 CR_BYTE = ord('\r')
+LF_BYTE = ord('\n')
 
 
 def _import_sendfile():
@@ -1031,29 +1032,42 @@ def __init__(self, file, type):
         self.file = file
         self.type = type
         self._prev_chunk_endswith_cr = False
-        if type == 'a' and os.linesep != '\r\n':
+        if type == 'a':
             self._data_wrapper = self._posix_ascii_data_wrapper
         else:
             self._data_wrapper = None
 
     def _posix_ascii_data_wrapper(self, chunk):
         """The data wrapper used for sending data in ASCII mode on
         systems using a single line terminator, handling those cases
-        where CRLF ('\r\n') gets delivered in two chunks.
+        where CRLF ('\r\n') gets delivered in two chunks as well as
+        cased where a file has mixed line endings.
         """
         chunk = bytearray(chunk)
         pos = 0
-        if self._prev_chunk_endswith_cr and chunk.startswith(b'\n'):
+        if self._prev_chunk_endswith_cr:
+            if len(chunk) == 0 or chunk[pos] != LF_BYTE:
+                chunk.insert(pos, LF_BYTE)
             pos += 1
+        self._prev_chunk_endswith_cr = False
         while True:
-            pos = chunk.find(b'\n', pos)
-            if pos == -1:
-                break
-            if chunk[pos - 1] != CR_BYTE:
-                chunk.insert(pos, CR_BYTE)
+            cr_pos = chunk.find(b'\r', pos)
+            lf_pos = chunk.find(b'\n', pos)
+            if cr_pos != -1 and (lf_pos == -1 or cr_pos < lf_pos):
+                if cr_pos == len(chunk) - 1:
+                    self._prev_chunk_endswith_cr = True
+                    break
+                if chunk[cr_pos + 1] != LF_BYTE:
+                    chunk.insert(cr_pos + 1, LF_BYTE)
+                    pos = cr_pos + 1
                 pos += 1
-            pos += 1
-        self._prev_chunk_endswith_cr = chunk.endswith(b'\r')
+            elif lf_pos != -1 and (cr_pos == -1 or lf_pos < cr_pos):
+                if chunk[lf_pos - 1] != CR_BYTE:
+                    chunk.insert(lf_pos, CR_BYTE)
+                    pos = lf_pos + 1
+                pos += 1
+            else:
+                break
         return chunk
 
     def more(self):

pyftpdlib/test/test_functional.py

@@ -1018,19 +1018,44 @@ def test_retr(self):
                           "retr " + bogus, lambda x: x)
 
     def test_retr_ascii(self):
-        """Test RETR in ASCII mode."""
+        """Test ASCII mode RETR for data without line endings."""
 
-        data = (b'abcde12345' + b(os.linesep)) * 100000
+        data = b'abcde12345' * 100000
+        self.file.write(data)
+        self.file.close()
+        self.retrieve_ascii("retr " + TESTFN, self.dummyfile.write)
+        self.dummyfile.seek(0)
+        datafile = self.dummyfile.read()
+        self.assertEqual(len(data), len(datafile))
+        self.assertEqual(hash(data), hash(datafile))
+
+    def test_retr_ascii_cr(self):
+        """Test ASCII mode RETR for data with CR line endings."""
+
+        data = b'abcde12345\r' * 100000
+        self.file.write(data)
+        self.file.close()
+        self.retrieve_ascii("retr " + TESTFN, self.dummyfile.write)
+        expected = data.replace(b'\r', b'\r\n')
+        self.dummyfile.seek(0)
+        datafile = self.dummyfile.read()
+        self.assertEqual(len(expected), len(datafile))
+        self.assertEqual(hash(expected), hash(datafile))
+
+    def test_retr_ascii_lf(self):
+        """Test ASCII mode RETR for data with LF line endings."""
+
+        data = b'abcde12345\n' * 100000
         self.file.write(data)
         self.file.close()
         self.retrieve_ascii("retr " + TESTFN, self.dummyfile.write)
-        expected = data.replace(b(os.linesep), b'\r\n')
+        expected = data.replace(b'\n', b'\r\n')
         self.dummyfile.seek(0)
         datafile = self.dummyfile.read()
         self.assertEqual(len(expected), len(datafile))
         self.assertEqual(hash(expected), hash(datafile))
 
-    def test_retr_ascii_already_crlf(self):
+    def test_retr_ascii_crlf(self):
         """Test ASCII mode RETR for data with CRLF line endings."""
 
         data = b'abcde12345\r\n' * 100000
@@ -1042,6 +1067,19 @@ def test_retr_ascii_already_crlf(self):
         self.assertEqual(len(data), len(datafile))
         self.assertEqual(hash(data), hash(datafile))
 
+    def test_retr_ascii_mixed_eol(self):
+        """Test ASCII mode RETR for data with mixed line endings."""
+
+        data = b'abcde12345\r\nabcde12345\nabcde12345\r' * 100000
+        self.file.write(data)
+        self.file.close()
+        self.retrieve_ascii("retr " + TESTFN, self.dummyfile.write)
+        expected = b'abcde12345\r\nabcde12345\r\nabcde12345\r\n' * 100000
+        self.dummyfile.seek(0)
+        datafile = self.dummyfile.read()
+        self.assertEqual(len(expected), len(datafile))
+        self.assertEqual(hash(expected), hash(datafile))
+
     @retry_on_failure()
     def test_restore_on_retr(self):
         data = b'abcde12345' * 1000000