From 35e29a53ee3572ac69aad93a1af307c0b6703774 Mon Sep 17 00:00:00 2001
From: Andrey Kislyuk <kislyuk@gmail.com>
Date: Sun, 1 Apr 2018 21:51:35 -0700
Subject: [PATCH 1/4] WIP

---
 hca/dss/__init__.py | 26 ++++++++++++--------------
 1 file changed, 12 insertions(+), 14 deletions(-)

diff --git a/hca/dss/__init__.py b/hca/dss/__init__.py
index 84a14231..d715ae60 100644
--- a/hca/dss/__init__.py
+++ b/hca/dss/__init__.py
@@ -6,6 +6,7 @@
 from io import open
 
 import requests
+from requests.packages.urllib3.exceptions import ProtocolError, DecodeError, ReadTimeoutError, MaxRetryError
 
 from ..util import SwaggerClient
 from ..util.exceptions import SwaggerAPIException
@@ -37,24 +38,21 @@ def download(self, bundle_uuid, replica, version="", dest_name=""):
             file_uuid = file_["uuid"]
             filename = file_.get("name", file_uuid)
 
-            logger.info("File %s: Retrieving...", filename)
-            response = self.get_file._request(dict(uuid=file_uuid, replica=replica), stream=True)
+            logger.info("Retrieving file %s", filename)
 
+            file_path = os.path.join(dest_name, filename)
+
+            retries = self.get_session().adapters["https://"].max_retries
+            # When streaming response data, requests/urllib3 does not obey its usual retry logic, so we reenact it here.
             try:
-                if response.ok:
-                    file_path = os.path.join(dest_name, filename)
+                with self.get_file.stream(uuid=file_uuid, replica=replica) as fh, open(file_path, "wb") as dest_fh:
                     logger.info("%s", "File {}: GET SUCCEEDED. Writing to disk.".format(filename))
-                    with open(file_path, "wb") as fh:
-                        for chunk in response.iter_content(chunk_size=1024*1024):
-                            if chunk:
-                                fh.write(chunk)
+                    while True:
+                        dest_fh.write(fh.raw.read(1024*1024))
                     logger.info("%s", "File {}: GET SUCCEEDED. Stored at {}.".format(filename, file_path))
-
-                else:
-                    logger.error("%s", "File {}: GET FAILED.".format(filename))
-                    logger.error("%s", "Response: {}".format(response.text))
-            finally:
-                response.close()
+            except (ProtocolError, DecodeError, ReadTimeoutError) as e:
+                logger.error(e)
+                retries = retries.increment(method="GET", error=e)
         return {}
 
     def upload(self, src_dir, replica, staging_bucket, timeout_seconds=1200):

From 8f4c0cd52eea208bd38e21dd812d1f464d75c769 Mon Sep 17 00:00:00 2001
From: Andrey Kislyuk <kislyuk@gmail.com>
Date: Sun, 1 Apr 2018 22:39:44 -0700
Subject: [PATCH 2/4] WIP

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index b949a381..20ef66e0 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,7 @@
 SHELL=/bin/bash
 
 test: lint install
-	coverage run --source=$$(python setup.py --name) -m unittest discover
+	coverage run --source=$$(python setup.py --name) -m unittest discover -v
 
 lint:
 	./setup.py flake8

From baacc6d5848bbf9ca2c0be12f2ae90a08c925829 Mon Sep 17 00:00:00 2001
From: Andrey Kislyuk <kislyuk@gmail.com>
Date: Mon, 2 Apr 2018 08:54:24 -0700
Subject: [PATCH 3/4] WIP

---
 hca/dss/__init__.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/hca/dss/__init__.py b/hca/dss/__init__.py
index d715ae60..5c7d03c2 100644
--- a/hca/dss/__init__.py
+++ b/hca/dss/__init__.py
@@ -48,7 +48,10 @@ def download(self, bundle_uuid, replica, version="", dest_name=""):
                 with self.get_file.stream(uuid=file_uuid, replica=replica) as fh, open(file_path, "wb") as dest_fh:
                     logger.info("%s", "File {}: GET SUCCEEDED. Writing to disk.".format(filename))
                     while True:
-                        dest_fh.write(fh.raw.read(1024*1024))
+                        chunk = fh.raw.read(1024*1024)
+                        if len(chunk) == 0:
+                            break
+                        dest_fh.write(chunk)
                     logger.info("%s", "File {}: GET SUCCEEDED. Stored at {}.".format(filename, file_path))
             except (ProtocolError, DecodeError, ReadTimeoutError) as e:
                 logger.error(e)

From 7e641f9dc3e09ea61f2ad0881de7e5b5db8a736e Mon Sep 17 00:00:00 2001
From: Andrey Kislyuk <kislyuk@gmail.com>
Date: Mon, 2 Apr 2018 12:39:08 -0700
Subject: [PATCH 4/4] Remove unused import

---
 hca/dss/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hca/dss/__init__.py b/hca/dss/__init__.py
index 5c7d03c2..ac5d7ef8 100644
--- a/hca/dss/__init__.py
+++ b/hca/dss/__init__.py
@@ -6,7 +6,7 @@
 from io import open
 
 import requests
-from requests.packages.urllib3.exceptions import ProtocolError, DecodeError, ReadTimeoutError, MaxRetryError
+from requests.packages.urllib3.exceptions import ProtocolError, DecodeError, ReadTimeoutError
 
 from ..util import SwaggerClient
 from ..util.exceptions import SwaggerAPIException