feature/https-or-http-func (#176)

* add try_url function, associated unit tests

* function as a parameter for try_url

* PR comments, move try_url to validators

* docstring touchup

* results from linter edits, use fstrings instead

cdp_backend/database/validators.py

@@ -195,3 +195,36 @@ def is_valid(value: str) -> bool:
         return value is None or value in get_all_class_attr_values(constant_cls)
 
     return is_valid
+
+
+def try_url(url: str, resolve_func: Callable = resource_exists) -> str:
+    """
+    Given a URL, return the URL with the protocol that exists (http or https)
+    with a preference for https.
+
+    Parameters
+    ----------
+    url: str
+        The target resource url.
+    resolve_func: func(url: str) -> bool
+        A function that takes in a str URL and determines whether it is reachable.
+        Default is our "resource_exists" func
+
+    Returns
+    -------
+    resource_url: str
+        The url with the correct protocol based on where the resource exists.
+        If does not exist, a LookupError is raised.
+    """
+    secure_url = url.replace("http://", "https://")
+    if resolve_func(secure_url):
+        return secure_url
+
+    if resolve_func(url):
+        return url
+
+    raise LookupError(f"the resource {url} could not be found")
+
+
+def is_secure_uri(url: str) -> bool:
+    return url.startswith("https://")

cdp_backend/pipeline/event_gather_pipeline.py

@@ -19,7 +19,7 @@
 from ..database import constants as db_constants
 from ..database import functions as db_functions
 from ..database import models as db_models
-from ..database.validators import resource_exists
+from ..database.validators import is_secure_uri, try_url
 from ..file_store import functions as fs_functions
 from ..sr_models import GoogleCloudSRModel, WebVTTSRModel
 from ..utils import constants_utils, file_utils
@@ -337,24 +337,22 @@ def convert_video_and_handle_host(
         video_filepath = mp4_filepath
 
     # Store if the original host isn't https
-    elif not session.video_uri.startswith("https://"):
-        # Attempt to find secure version of resource and simply swap
-        # otherwise we will have to host
-        if session.video_uri.startswith("http://"):
-            secure_uri = session.video_uri.replace("http://", "https://")
-            if resource_exists(secure_uri):
+    elif not is_secure_uri(session.video_uri):
+        try:
+            resource_uri = try_url(session.video_uri)
+        except LookupError:
+            # The provided URI could still be like GCS or S3 URI, which
+            # works for download but not for streaming / hosting
+            cdp_will_host = True
+        else:
+            if is_secure_uri(resource_uri):
                 log.info(
                     f"Found secure version of {session.video_uri}, "
                     f"updating stored video URI."
                 )
-                hosted_video_media_url = secure_uri
+                hosted_video_media_url = resource_uri
             else:
                 cdp_will_host = True
-
-        # The provided URI could still be like GCS or S3 URI, which works for download
-        # but not for streaming / hosting
-        else:
-            cdp_will_host = True
     else:
         hosted_video_media_url = session.video_uri
 

cdp_backend/tests/database/test_validators.py

@@ -193,3 +193,35 @@ def test_role_title_is_valid(title: str, expected_result: bool) -> None:
     validator_func = validators.create_constant_value_validator(RoleTitle)
     actual_result = validator_func(title)
     assert actual_result == expected_result
+
+
+@pytest.mark.parametrize(
+    "url, expected, exception",
+    [
+        (
+            "https://exists",
+            "https://exists",
+            None,
+        ),
+        (
+            "http://exists",
+            "https://exists",
+            None,
+        ),
+        (
+            "ftp://some-ftp-url",
+            "",
+            LookupError,
+        ),
+    ],
+)
+def test_try_url_no_exceptions(url: str, expected: str, exception: Exception) -> None:
+    if exception is None:
+        assert validators.try_url(url, mock_resource_exists) == expected
+    else:
+        with pytest.raises(exception):
+            assert validators.try_url(url, mock_resource_exists) == expected
+
+
+def mock_resource_exists(url: str) -> bool:
+    return url == "https://exists"