Merge pull request fsspec#718 from krisgeus/issue-717-fix-error-handl…

…ing-file-size Issue 717 fix error handling file size
earthobservations · Jul 30, 2021 · cf766f5 · cf766f5
2 parents 93e5a68 + 2861182
commit cf766f5
Show file tree

Hide file tree

Showing 2 changed files with 32 additions and 15 deletions.
diff --git a/fsspec/implementations/http.py b/fsspec/implementations/http.py
@@ -573,13 +573,16 @@ def close(self):
         pass
 
     def __reduce__(self):
-        return reopen, (
-            self.fs,
-            self.url,
-            self.mode,
-            self.blocksize,
-            self.cache.name,
-            self.size,
+        return (
+            reopen,
+            (
+                self.fs,
+                self.url,
+                self.mode,
+                self.blocksize,
+                self.cache.name,
+                self.size,
+            ),
         )
 
 
@@ -671,13 +674,20 @@ async def _file_size(url, session=None, size_policy="head", **kwargs):
     else:
         raise TypeError('size_policy must be "head" or "get", got %s' "" % size_policy)
     async with r:
-        # TODO:
-        #  recognise lack of 'Accept-Ranges', or  'Accept-Ranges': 'none' (not 'bytes')
-        #  to mean streaming only, no random access => return None
-        if "Content-Length" in r.headers:
-            return int(r.headers["Content-Length"])
-        elif "Content-Range" in r.headers:
-            return int(r.headers["Content-Range"].split("/")[1])
+        try:
+            r.raise_for_status()
+
+            # TODO:
+            #  recognise lack of 'Accept-Ranges',
+            #                 or 'Accept-Ranges': 'none' (not 'bytes')
+            #  to mean streaming only, no random access => return None
+            if "Content-Length" in r.headers:
+                return int(r.headers["Content-Length"])
+            elif "Content-Range" in r.headers:
+                return int(r.headers["Content-Range"].split("/")[1])
+        except aiohttp.ClientResponseError:
+            logger.debug("Error retrieving file size")
+            return None
         r.close()
 
 

diff --git a/fsspec/implementations/tests/test_http.py b/fsspec/implementations/tests/test_http.py
@@ -66,10 +66,15 @@ def do_GET(self):
             self._respond(200, data=d)
 
     def do_HEAD(self):
+        d = data if self.path == "/index/realfile" else index
+        if "head_not_auth" in self.headers:
+            self._respond(
+                403, {"Content-Length": 123}, b"not authorized for HEAD request"
+            )
+            return
         if "head_ok" not in self.headers:
             self._respond(405)
             return
-        d = data if self.path == "/index/realfile" else index
         if self.path.rstrip("/") not in ["/index/realfile", "/index"]:
             self._respond(404)
         elif "give_length" in self.headers:
@@ -286,6 +291,8 @@ def test_methods(server):
         {"give_length": "true"},
         {"give_length": "true", "head_ok": "true"},
         {"give_range": "true"},
+        {"give_length": "true", "head_not_auth": "true"},
+        {"give_range": "true", "head_not_auth": "true"},
     ],
 )
 def test_random_access(server, headers):