From 968e74890ded0a3fef2550f1c4c68daaab80842b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Santos?= Date: Thu, 16 Jan 2020 14:21:59 +0000 Subject: [PATCH 1/4] pull: treat HTTP redirects without Location header as error Fixes #2963 --- dvc/remote/http.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/dvc/remote/http.py b/dvc/remote/http.py index caa094b977..6879e1436f 100644 --- a/dvc/remote/http.py +++ b/dvc/remote/http.py @@ -101,7 +101,14 @@ def _request(self, method, url, **kwargs): kwargs.setdefault("timeout", self.REQUEST_TIMEOUT) try: - return self._session.request(method, url, **kwargs) + res = self._session.request(method, url, **kwargs) + + if kwargs["allow_redirects"] and res.is_redirect: + # AWS s3 doesn't like to add a location header to its redirects + # from https://s3.amazonaws.com//* type URLs. + # This should be treated as an error + raise requests.exceptions.RequestException + except requests.exceptions.RequestException: raise DvcException("could not perform a {} request".format(method)) From d3f925ae635bfe593be749ca9e0c01d194cfe58d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Santos?= Date: Thu, 16 Jan 2020 14:30:03 +0000 Subject: [PATCH 2/4] add missing return --- dvc/remote/http.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dvc/remote/http.py b/dvc/remote/http.py index 6879e1436f..e76185d5d6 100644 --- a/dvc/remote/http.py +++ b/dvc/remote/http.py @@ -109,6 +109,8 @@ def _request(self, method, url, **kwargs): # This should be treated as an error raise requests.exceptions.RequestException + return res + except requests.exceptions.RequestException: raise DvcException("could not perform a {} request".format(method)) From c5eb13c114821b02c7a29546261c7611d91d126d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Santos?= Date: Thu, 16 Jan 2020 14:33:54 +0000 Subject: [PATCH 3/4] update redirect check --- dvc/remote/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/remote/http.py b/dvc/remote/http.py index e76185d5d6..ca68cac5b1 100644 --- a/dvc/remote/http.py +++ b/dvc/remote/http.py @@ -103,7 +103,7 @@ def _request(self, method, url, **kwargs): try: res = self._session.request(method, url, **kwargs) - if kwargs["allow_redirects"] and res.is_redirect: + if kwargs["allow_redirects"] and res.status_code in (301, 302): # AWS s3 doesn't like to add a location header to its redirects # from https://s3.amazonaws.com//* type URLs. # This should be treated as an error From df22dd4defe804121d7dc844ac6fb3442eb7ed45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20Santos?= Date: Thu, 16 Jan 2020 17:58:44 +0000 Subject: [PATCH 4/4] specifically look for Location header in response --- dvc/remote/http.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/dvc/remote/http.py b/dvc/remote/http.py index ca68cac5b1..c04256a462 100644 --- a/dvc/remote/http.py +++ b/dvc/remote/http.py @@ -103,7 +103,13 @@ def _request(self, method, url, **kwargs): try: res = self._session.request(method, url, **kwargs) - if kwargs["allow_redirects"] and res.status_code in (301, 302): + redirect_no_location = ( + kwargs["allow_redirects"] + and res.status_code in (301, 302) + and "location" not in res.headers + ) + + if redirect_no_location: # AWS s3 doesn't like to add a location header to its redirects # from https://s3.amazonaws.com//* type URLs. # This should be treated as an error