tests: fix tests and introduce ret_reason for availability check

dclab/rtdc_dataset/fmt_http.py

@@ -277,7 +277,7 @@ def is_available(self):
             # don't even bother
             self._available_verified = False
         if self._available_verified is None:
-            avail, reason = is_url_available(self.location)
+            avail, reason = is_url_available(self.location, ret_reason=True)
             if reason in ["forbidden", "not found"]:
                 # we cannot access the URL in the near future
                 self._available_verified = False
@@ -286,13 +286,15 @@ def is_available(self):
         return self._available_verified
 
 
-def is_url_available(url: str):
+def is_url_available(url: str, ret_reason=False):
     """Check whether a URL is available
 
     Parameters
     ----------
     url: str
         full URL to the object
+    ret_reason: bool
+        whether to return reason for unavailability
 
     Returns
     -------
@@ -325,7 +327,12 @@ def is_url_available(url: str):
                 except OSError:
                     reason = "oserror"
                     pass
-    return avail, reason
+    else:
+        reason = "invalid"
+    if ret_reason:
+        return avail, reason
+    else:
+        return avail
 
 
 @functools.lru_cache()

dclab/rtdc_dataset/load.py

@@ -51,7 +51,7 @@ def new_dataset(data, identifier=None, **kwargs):
     elif fmt_dcor.is_dcor_url(data):
         return fmt_dcor.RTDC_DCOR(data, identifier=identifier, **kwargs)
     elif fmt_http.is_http_url(data):
-        if fmt_http.is_url_available(data):
+        if fmt_http.is_url_available(data, ret_reason=False):
             return fmt_http.RTDC_HTTP(data, identifier=identifier)
         elif fmt_s3.is_s3_url(data):
             return fmt_s3.RTDC_S3(data, identifier=identifier, **kwargs)

tests/test_rtdc_fmt_http.py

@@ -57,16 +57,16 @@ def test_identifier():
 @pytest.mark.parametrize("url, avail", [
     ("https://objectstore.hpccloud.mpcdf.mpg.de/"
      "circle-5a7a053d-55fb-4f99-960c-f478d0bd418f/"
-     "resource/fb7/19f/b2-bd9f-817a-7d70-f4002af916f0", True),
+     "resource/fb7/19f/b2-bd9f-817a-7d70-f4002af916f0", (True, "none")),
     # "noexisting"
     ("https://objectstore.hpccloud.mpcdf.mpg.de/"
      "noexisting-5a7a053d-55fb-4f99-960c-f478d0bd418f/"
-     "resource/fb7/19f/b2-bd9f-817a-7d70-f4002af916f0", False),
+     "resource/fb7/19f/b2-bd9f-817a-7d70-f4002af916f0", (False, "not found")),
     # invalid URL
-    ("https://example.com", False),
+    ("https://example.com", (False, "invalid")),
     # nonexistent host
-    (f"http://{uuid.uuid4()}.com/bucket/resource", False),
-    (f"https://{uuid.uuid4()}.com/bucket/resource", False),
+    (f"http://{uuid.uuid4()}.com/bucket/resource", (False, "no connection")),
+    (f"https://{uuid.uuid4()}.com/bucket/resource", (False, "no connection")),
 ])
 def test_object_available(url, avail):
     act = is_url_available(url)