Add FTP downloader

pooch/core.py

@@ -8,9 +8,9 @@
 from warnings import warn
 
 import requests
-
-from .utils import file_hash, check_version
-from .downloaders import HTTPDownloader
+from requests_ftp.ftp import FTPSession
+from .utils import file_hash, check_version, infer_protocol_options
+from .downloaders import HTTPDownloader, FTPDownloader
 
 
 def create(
@@ -341,7 +341,7 @@ def mydownloader(url, output_file, pooch):
             os.makedirs(str(self.abspath))
 
         full_path = self.abspath / fname
-
+        url = self.get_url(fname)
         in_storage = full_path.exists()
         if not in_storage:
             action = "download"
@@ -357,15 +357,20 @@ def mydownloader(url, output_file, pooch):
                     action_word[action], fname, self.get_url(fname), str(self.path)
                 )
             )
+
             if downloader is None:
-                downloader = HTTPDownloader()
+                options = infer_protocol_options(url)
+                if options["protocol"] == "ftp":
+                    downloader = FTPDownloader()
+                else:
+                    downloader = HTTPDownloader()
             # Stream the file to a temporary so that we can safely check its hash before
             # overwriting the original
             tmp = tempfile.NamedTemporaryFile(delete=False, dir=str(self.abspath))
             # Close the temp file so that the downloader can decide how to opened it
             tmp.close()
             try:
-                downloader(self.get_url(fname), tmp.name, self)
+                downloader(url, tmp.name, self)
                 self._check_download_hash(fname, tmp.name)
                 # Ensure the parent directory exists in case the file is in a
                 # subdirectory. Otherwise, move will cause an error.
@@ -481,5 +486,10 @@ def is_available(self, fname):
         """
         self._assert_file_in_registry(fname)
         source = self.get_url(fname)
-        response = requests.head(source, allow_redirects=True)
+        options = infer_protocol_options(source)
+        if options["protocol"] == "ftp":
+            session = FTPSession()
+        else:
+            session = requests
+        response = session.head(source, allow_redirects=True)
         return bool(response.status_code == 200)

pooch/downloaders.py

@@ -4,6 +4,7 @@
 import sys
 
 import requests
+from requests_ftp.ftp import FTPSession
 
 try:
     from tqdm import tqdm
@@ -149,3 +150,58 @@ def __call__(self, url, output_file, pooch):
         finally:
             if ispath:
                 output_file.close()
+
+
+class FTPDownloader(HTTPDownloader):
+    def __call__(self, url, output_file, pooch):
+        """
+        Download the given URL over FTP to the given output file.
+
+        Uses :func:`requests_ftp.ftp.FTPSession().get`.
+
+        Parameters
+        ----------
+        url : str
+            The URL to the file you want to download.
+        output_file : str or file-like object
+            Path (and file name) to which the file will be downloaded.
+        pooch : :class:`~pooch.Pooch`
+            The instance of :class:`~pooch.Pooch` that is calling this method.
+        """
+
+        kwargs = self.kwargs.copy()
+        kwargs.setdefault("stream", True)
+        ispath = not hasattr(output_file, "write")
+        if ispath:
+            output_file = open(output_file, "w+b")
+        try:
+            session = FTPSession()
+            response = session.get(url, **kwargs)
+            response.raise_for_status()
+            content = response.iter_content(chunk_size=self.chunk_size)
+            if self.progressbar:
+                total = int(response.headers.get("content-length", 0))
+                use_ascii = bool(sys.platform == "win32")
+                progress = tqdm(
+                    total=total,
+                    ncols=79,
+                    ascii=use_ascii,
+                    unit="B",
+                    unit_scale=True,
+                    leave=True,
+                )
+            for chunk in content:
+                if chunk:
+                    output_file.write(chunk)
+                    output_file.flush()
+                    if self.progressbar:
+                        progress.update(self.chunk_size)
+
+            if self.progressbar:
+                progress.reset()
+                progress.update(total)
+                progress.close()
+
+        finally:
+            if ispath:
+                output_file.close()

pooch/tests/test_utils.py

@@ -6,7 +6,7 @@
 from tempfile import NamedTemporaryFile
 
 from ..core import Pooch
-from ..utils import make_registry
+from ..utils import make_registry, infer_protocol_options
 from .utils import check_tiny_data
 
 DATA_DIR = str(Path(__file__).parent / "data" / "store")
@@ -60,3 +60,12 @@ def test_registry_builder_recursive():
         check_tiny_data(pup.fetch("subdir/tiny-data.txt"))
     finally:
         os.remove(outfile.name)
+
+
+def test_infer_protocol_options():
+    "Infer protocol options from URL"
+    url = "http://127.0.0.1:8080/test.nc"
+    assert infer_protocol_options(url) == {"protocol": "http", "path": url}
+
+    url = "ftp://127.0.0.1:8080/test.nc"
+    assert infer_protocol_options(url) == {"protocol": "ftp", "path": url}

pooch/utils.py

@@ -3,7 +3,7 @@
 """
 from pathlib import Path
 import hashlib
-
+from urllib.parse import urlsplit
 import appdirs
 from packaging.version import Version
 
@@ -160,3 +160,21 @@ def make_registry(directory, output, recursive=True):
             # Only use Unix separators for the registry so that we don't go insane
             # dealing with file paths.
             outfile.write("{} {}\n".format(fname.replace("\\", "/"), fhash))
+
+
+def infer_protocol_options(urlpath):
+    """
+    Infer protocol options from URL path.
+
+    Parameters
+    ----------
+    urlpath : str or unicode
+        URL (e.g.: http://127.0.0.1:8080/test.nc, ftp://127.0.0.1:8080/test.nc)
+
+    Returns
+    -------
+    Options dict
+    """
+    parsed_path = urlsplit(urlpath)
+    protocol = parsed_path.scheme or "file"
+    return {"protocol": protocol, "path": urlpath}

requirements.txt

@@ -1,3 +1,4 @@
 requests
 packaging
 appdirs
+requests-ftp