Updated to requests instead of urllib

Makefile

@@ -39,14 +39,14 @@ dlginstall:	  ## Install into $DLG_ROOT/code
 .PHONY: fmt
 fmt:              ## Format code using black & isort.
 	$(ENV_PREFIX)isort dlg_example_cmpts/
-	$(ENV_PREFIX)black -l 79 dlg_example_cmpts/
-	$(ENV_PREFIX)black -l 79 tests/
+	$(ENV_PREFIX)black -l 90 dlg_example_cmpts/
+	$(ENV_PREFIX)black -l 90 tests/
 
 .PHONY: lint
 lint:             ## Run pep8, black, mypy linters.
-	$(ENV_PREFIX)flake8 dlg_example_cmpts/
-	$(ENV_PREFIX)black -l 79 --check dlg_example_cmpts/
-	$(ENV_PREFIX)black -l 79 --check tests/
+	$(ENV_PREFIX)flake8 -l 90 dlg_example_cmpts/
+	$(ENV_PREFIX)black -l 90 --check dlg_example_cmpts/
+	$(ENV_PREFIX)black -l 90 --check tests/
 	$(ENV_PREFIX)mypy --ignore-missing-imports dlg_example_cmpts/
 
 .PHONY: test

dlg_example_cmpts/apps.py

@@ -16,11 +16,13 @@
 import logging
 import os
 import pickle
-import requests
 from glob import glob
+from urllib.request import url2pathname
 
 import numpy as np
+import requests  # type: ignore
 from dlg import droputils
+from dlg.meta import dlg_int_param, dlg_string_param
 
 try:
     from dlg.drop import BarrierAppDROP
@@ -32,10 +34,10 @@
 except ImportError:
     from dlg.apps.branch import BranchAppDrop
 
-from dlg.meta import dlg_int_param, dlg_string_param
 
 logger = logging.getLogger(__name__)
 
+
 ##
 # @brief MyBranch
 # @details Simple app to demonstrate how to write a branch actually making a
@@ -212,9 +214,7 @@ def run(self):
         if len(self.value) == 0:
             logger.warning(f"No matching files found for {filetmpl}")
         else:
-            logger.info(
-                f"Number of files found matching {filetmpl}: {len(self.value)}"
-            )
+            logger.info(f"Number of files found matching {filetmpl}: {len(self.value)}")
         self.writeData()
 
 
@@ -269,6 +269,57 @@ def run(self):
         self.writeData()
 
 
+class LocalFileAdapter(requests.adapters.BaseAdapter):
+    """Helper protocol Adapter to allow Requests to GET file:// URLs"""
+
+    @staticmethod
+    def _chkpath(method, path):
+        """Return an HTTP status for the given filesystem path."""
+        if method.lower() in ("put", "delete"):
+            return 501, "Not Implemented"  # TODO
+        elif method.lower() not in ("get", "head"):
+            return 405, "Method Not Allowed"
+        elif os.path.isdir(path):
+            return 400, "Path Not A File"
+        elif not os.path.isfile(path):
+            return 404, "File Not Found"
+        elif not os.access(path, os.R_OK):
+            return 403, "Access Denied"
+        else:
+            return 200, "OK"
+
+    def send(self, req, **kwargs):  # pylint: disable=unused-argument
+        """Return the file specified by the given request
+
+        @type req: C{PreparedRequest}
+        @todo: Should I bother filling `response.headers` and processing
+               If-Modified-Since and friends using `os.stat`?
+        """
+        path = os.path.normcase(os.path.normpath(url2pathname(req.path_url)))
+        response = requests.Response()
+
+        response.status_code, response.reason = self._chkpath(req.method, path)
+        if response.status_code == 200 and req.method.lower() != "head":
+            try:
+                response.raw = open(path, "rb")
+            except (OSError, IOError) as err:
+                response.status_code = 500
+                response.reason = str(err)
+
+        if isinstance(req.url, bytes):
+            response.url = req.url.decode("utf-8")
+        else:
+            response.url = req.url
+
+        response.request = req
+        response.connection = self
+
+        return response
+
+    def close(self):
+        pass
+
+
 ##
 # @brief AdvUrlRetrieve
 # @details An APP that retrieves the content of a URL and writes
@@ -342,9 +393,12 @@ def readData(self):
             urlParts.update({name: part})
         self.urlParts = urlParts
         self.url = self.constructUrl()
+        requests_session = requests.session()
+        if "file://" == self.url[:7]:
+            requests_session.mount("file://", LocalFileAdapter())
         logger.info("Reading from %s", self.url)
         try:
-            u = requests.get(self.url)
+            u = requests_session.get(self.url)
         except Exception as e:
             raise e
         # finally read the content of the URL
@@ -368,9 +422,7 @@ def writeData(self):
                 logger.warning(f"Output with name {output.name} ignored!")
 
         if not written:
-            raise TypeError(
-                "No matching output drop found." + "Nothing written"
-            )
+            raise TypeError("No matching output drop found." + "Nothing written")
 
     def run(self):
         self.readData()
@@ -413,9 +465,7 @@ def writeData(self, data):
             output.write(d)
 
     def run(self):
-        if (
-            len(self.inputs) == 0
-        ):  # if there is no input use the argument value
+        if len(self.inputs) == 0:  # if there is no input use the argument value
             try:
                 logger.debug("Input found: %s", self.string)
                 data = json.loads(self.string)

tests/test_cmpts.py

@@ -1,10 +1,11 @@
 """
 Test module for example components.
 """
+
 from glob import glob
 import os
 import pickle
-import urllib
+import requests
 import http
 import logging
 import json
@@ -106,9 +107,7 @@ def _runLengthTest(self, arrayDROP):
         n = InMemoryDROP("n", "n")  # the memory drop for the NO branch
         y = InMemoryDROP("y", "y")  # the memory drop for the YES branch
         if not isinstance(arrayDROP, InMemoryDROP):
-            startDrop = NullDROP(
-                "i", "i"
-            )  # just to be able to start the execution
+            startDrop = NullDROP("i", "i")  # just to be able to start the execution
             m = InMemoryDROP("m", "m")  # the receiving drop
             startDrop.addConsumer(arrayDROP)
             m.addProducer(arrayDROP)
@@ -186,9 +185,7 @@ def test_LengthCheck_class_zero(self):
         self.assertEqual(oid, "n")
         self.assertEqual(check, length)
 
-    @pytest.mark.filterwarnings(
-        "ignore::pytest.PytestUnhandledThreadExceptionWarning"
-    )
+    @pytest.mark.filterwarnings("ignore::pytest.PytestUnhandledThreadExceptionWarning")
     def test_LengthCheck_wrongType(self):
         # check wrong type
         w = InMemoryDROP("w", "w")
@@ -363,7 +360,7 @@ def test_AdvUrlRetrieve_wrongUrl(self):
         e.addOutput(o)
         with droputils.DROPWaiterCtx(self, o, timeout=10):
             a.setCompleted()
-        self.assertRaises(urllib.error.URLError)
+        self.assertRaises(requests.exceptions.RequestException)
 
     def test_AdvUrlRetrieve_invalidUrl(self):
         """
@@ -405,7 +402,7 @@ def test_AdvUrlRetrieve_noOutput(self):
 
         e.addInput(a)
         # e.addOutput(o)
-        with droputils.DROPWaiterCtx(self, e, timeout=10):
+        with droputils.DROPWaiterCtx(self, e, timeout=2):
             a.setCompleted()
         # content = json.loads(pickle.loads(droputils.allDropContents(a)))
         # self.assertEqual(content["args"], testContent["args"])

tests/test_funcs.py

@@ -1,8 +1,6 @@
 import pytest, unittest
 import os
 import pickle
-import urllib
-import http
 
 from glob import glob
 from dlg import droputils