Merge pull request #8 from gautamc/master

adding function to download large files without running out of memory.

examples/ftpclient.hs

@@ -0,0 +1,18 @@
+import Network.FTP.Client
+
+main :: IO()
+main = do
+  -- connect to SEC's FTP server
+  (ftp_handle, _) <- connectFTP "ftp.sec.gov" 21
+  -- Login as anonymous
+  login ftp_handle "anonymous" (Just "your.email@someserver.org") Nothing
+  -- Set transfer mode to binary
+  sendcmd ftp_handle "TYPE I"
+  -- Change to the directory where the file is located
+  cwd ftp_handle "/edgar/data/1251417/"
+  -- Download the file
+  downloadlargebinary ftp_handle "0001047469-14-010110.txt"
+  -- If we tried using downloadbinary then we might see that the program takes up
+  -- lot of memory and is eventually killed by the haskell runtime
+  -- downloadbinary ftp_handle "0001047469-14-010110.txt"
+  return ()

ftphs.cabal

@@ -44,7 +44,7 @@ Library
    UndecidableInstances, CPP, ScopedTypeVariables
   Build-Depends: network, parsec, base >= 3 && < 5,
                  mtl, regex-compat, 
-               hslogger, MissingH>=1.0.0
+               hslogger, MissingH>=1.0.0, bytestring
   GHC-Options: -O2
 
 Executable runtests

src/Network/FTP/Client.hs

@@ -207,7 +207,7 @@ module Network.FTP.Client(-- * Establishing\/Removing connections
                                    nlst, dir, 
                                    -- * File downloads
                                    getlines, getbinary,
-                                   downloadbinary,
+                                   downloadbinary, downloadlargebinary,
                                    -- * File uploads
                                    putlines, putbinary,
                                    uploadbinary,
@@ -231,6 +231,7 @@ import System.IO.Unsafe
 import System.Log.Logger
 import Network.Utils
 import Data.String.Utils
+import Data.ByteString (hGet, hPut)
 data FTPConnection = FTPConnection {readh :: IO String,
                                     writeh :: Handle,
                                     socket_internal :: Socket,
@@ -449,6 +450,30 @@ downloadbinary h fn = do (r0, r1) <- getbinary h fn
                          writeBinaryFile fn r0
                          return r1
 
+{- | Similar to downloadbinary, but downloads the file in blocks of 4096 bytes
+so that memory usage is limited when downloading large files.
+Uses Data.ByteString's hGet to read data from the socket and hPut to write data
+to the file, since it is more space and time efficient than String. -}
+downloadlargebinary :: FTPConnection -> FilePath -> IO FTPResult
+downloadlargebinary h fn = do
+  ftp_data_h <- transfercmd h $ "RETR " ++ fn
+  out_file_fh <- openFile fn WriteMode
+  getAndWrite ftp_data_h out_file_fh
+  getresp h
+  where
+    getAndWrite ftp_data_h out_file_fh = do
+      eof_p <- hIsEOF ftp_data_h
+      case eof_p of
+        True -> do
+          hClose ftp_data_h
+          hFlush out_file_fh
+          hClose out_file_fh
+          return ()
+        False -> do
+          buf <- hGet ftp_data_h 4096
+          hPut out_file_fh buf
+          getAndWrite ftp_data_h out_file_fh
+
 {- | Retrieves a list of files in the given directory. 
 
 FIXME: should this take a list of dirs? -}