fix for json response files key. Use requests Session with retries in…

… WebBackend. timeout on a requests call. code format

circup/__init__.py

@@ -264,8 +264,12 @@ def __init__(
         """
         self.name = name
         self.backend = backend
-        self.path = urljoin(backend.library_path,name, allow_fragments=False) if isinstance(backend,WebBackend) else os.path.join(backend.library_path, name)
-        url = urlparse(self.path,allow_fragments=False)
+        self.path = (
+            urljoin(backend.library_path, name, allow_fragments=False)
+            if isinstance(backend, WebBackend)
+            else os.path.join(backend.library_path, name)
+        )
+        url = urlparse(self.path, allow_fragments=False)
         if str(url.scheme).lower() in ("http", "https"):
             if url.path.endswith(".py") or url.path.endswith(".mpy"):
                 self.file = os.path.basename(url.path)
@@ -274,7 +278,9 @@ def __init__(
                 )
             else:
                 self.file = None
-                self.name = os.path.basename(url.path if url.path[:-1]=='/' else url.path[:-1])
+                self.name = os.path.basename(
+                    url.path if url.path[:-1] == "/" else url.path[:-1]
+                )
         else:
             if os.path.isfile(self.path):
                 # Single file module.
@@ -608,7 +614,9 @@ def find_modules(backend, bundles_list):
                     if not path.endswith(os.sep)
                     else path[:-1].split(os.sep)[-1] + os.sep
                 )
-                module_name = name if not path.find(os.sep) else module_name # should probably check for os.sep and use previous version if found
+                module_name = (
+                    name if not path.find(os.sep) else module_name
+                )  # should probably check for os.sep and use previous version if found
                 m = Module(
                     module_name,
                     backend,
@@ -1224,7 +1232,9 @@ def install(ctx, modules, pyext, requirement, auto, auto_file):  # pragma: no co
             auto_file = "code.py"
             print(f"Auto file: {auto_file}")
         # pass a local file with "./" or "../"
-        is_relative = not isinstance(ctx.obj["backend"], WebBackend) or auto_file.split(os.sep)[0] in [os.path.curdir, os.path.pardir]
+        is_relative = not isinstance(ctx.obj["backend"], WebBackend) or auto_file.split(
+            os.sep
+        )[0] in [os.path.curdir, os.path.pardir]
         if not os.path.isabs(auto_file) and not is_relative:
             auto_file = ctx.obj["backend"].get_file_path(auto_file or "code.py")
 
@@ -1301,7 +1311,6 @@ def uninstall(ctx, module):  # pragma: no cover
         continue
 
 
-
 # pylint: disable=too-many-branches
 
 

circup/backends.py

@@ -13,6 +13,7 @@
 from urllib.parse import urlparse, urljoin
 import click
 import requests
+from requests.adapters import HTTPAdapter
 from requests.auth import HTTPBasicAuth
 
 
@@ -116,7 +117,11 @@ def install_module(
                 click.echo("'{}' is already installed.".format(name))
                 return
 
-            library_path = os.path.join(device_path, self.LIB_DIR_PATH) if not isinstance(self,WebBackend) else urljoin(device_path,self.LIB_DIR_PATH)
+            library_path = (
+                os.path.join(device_path, self.LIB_DIR_PATH)
+                if not isinstance(self, WebBackend)
+                else urljoin(device_path, self.LIB_DIR_PATH)
+            )
 
             # Create the library directory first.
             self._create_library_directory(device_path, library_path)
@@ -206,6 +211,8 @@ def __init__(self, host, password, logger):
         self.password = password
         self.device_location = f"http://:{self.password}@{self.host}"
 
+        self.session = requests.Session()
+        self.session.mount(self.device_location, HTTPAdapter(max_retries=5))
         self.library_path = self.device_location + "/" + self.LIB_DIR_PATH
 
     def install_file_http(self, source):
@@ -215,19 +222,14 @@ def install_file_http(self, source):
         """
         file_name = source.split(os.path.sep)
         file_name = file_name[-2] if file_name[-1] == "" else file_name[-1]
-        target = (
-            self.device_location
-            + "/"
-            + self.LIB_DIR_PATH
-            + file_name
-        )
+        target = self.device_location + "/" + self.LIB_DIR_PATH + file_name
 
         auth = HTTPBasicAuth("", self.password)
         print(f"target: {target}")
         print(f"source: {source}")
 
         with open(source, "rb") as fp:
-            r = requests.put(target, fp.read(), auth=auth)
+            r = self.session.put(target, fp.read(), auth=auth)
             r.raise_for_status()
 
     def install_dir_http(self, source):
@@ -237,20 +239,15 @@ def install_dir_http(self, source):
         """
         mod_name = source.split(os.path.sep)
         mod_name = mod_name[-2] if mod_name[-1] == "" else mod_name[-1]
-        target = (
-            self.device_location
-            + "/"
-            + self.LIB_DIR_PATH
-            + mod_name
-        )
-        target = target + "/" if target[:-1]!="/" else target
+        target = self.device_location + "/" + self.LIB_DIR_PATH + mod_name
+        target = target + "/" if target[:-1] != "/" else target
         url = urlparse(target)
         auth = HTTPBasicAuth("", url.password)
         print(f"target: {target}")
         print(f"source: {source}")
 
         # Create the top level directory.
-        with requests.put(target, auth=auth) as r:
+        with self.session.put(target, auth=auth) as r:
             print(f"resp {r.content}")
             r.raise_for_status()
 
@@ -260,18 +257,36 @@ def install_dir_http(self, source):
             if rel_path == ".":
                 rel_path = ""
             for name in dirs:
-                path_to_create = urljoin( urljoin(target , rel_path + "/", allow_fragments=False) , name, allow_fragments=False) if rel_path != "" else urljoin(target , name, allow_fragments=False)
-                path_to_create = path_to_create + "/" if path_to_create[:-1]!="/" else path_to_create
+                path_to_create = (
+                    urljoin(
+                        urljoin(target, rel_path + "/", allow_fragments=False),
+                        name,
+                        allow_fragments=False,
+                    )
+                    if rel_path != ""
+                    else urljoin(target, name, allow_fragments=False)
+                )
+                path_to_create = (
+                    path_to_create + "/"
+                    if path_to_create[:-1] != "/"
+                    else path_to_create
+                )
                 print(f"dir_path_to_create: {path_to_create}")
-                with requests.put(path_to_create, auth=auth) as r:
+                with self.session.put(path_to_create, auth=auth) as r:
                     r.raise_for_status()
             for name in files:
                 with open(os.path.join(root, name), "rb") as fp:
-                    path_to_create = urljoin( urljoin(target , rel_path + "/", allow_fragments=False) , name, allow_fragments=False) if rel_path != "" else urljoin(target , name, allow_fragments=False)
+                    path_to_create = (
+                        urljoin(
+                            urljoin(target, rel_path + "/", allow_fragments=False),
+                            name,
+                            allow_fragments=False,
+                        )
+                        if rel_path != ""
+                        else urljoin(target, name, allow_fragments=False)
+                    )
                     print(f"file_path_to_create: {path_to_create}")
-                    with requests.put(
-                        path_to_create, fp.read(), auth=auth
-                    ) as r:
+                    with self.session.put(path_to_create, fp.read(), auth=auth) as r:
                         r.raise_for_status()
 
     def get_circuitpython_version(self):
@@ -283,7 +298,7 @@ def get_circuitpython_version(self):
         :return: A tuple with the version string for CircuitPython and the board ID string.
         """
         # pylint: disable=arguments-renamed
-        with requests.get(self.device_location + "/cp/version.json") as r:
+        with self.session.get(self.device_location + "/cp/version.json") as r:
             # pylint: disable=no-member
             if r.status_code != requests.codes.ok:
                 click.secho(
@@ -296,6 +311,7 @@ def get_circuitpython_version(self):
         return ver_json.get("version"), ver_json.get("board_id")
 
     def _get_modules(self, device_lib_path):
+        print(f"device_lib_path {device_lib_path}")
         return self._get_modules_http(device_lib_path)
 
     def _get_modules_http(self, url):
@@ -309,12 +325,17 @@ def _get_modules_http(self, url):
         result = {}
         u = urlparse(url)
         auth = HTTPBasicAuth("", u.password)
-        with requests.get(url, auth=auth, headers={"Accept": "application/json"}) as r:
+        with self.session.get(
+            url, auth=auth, headers={"Accept": "application/json"}
+        ) as r:
             r.raise_for_status()
 
             directory_mods = []
             single_file_mods = []
-            for entry in r.json():
+
+            for entry in r.json()["files"]:
+                # print(f"type: {type(entry)}")
+                # print(f"val: {entry}")
                 entry_name = entry.get("name")
                 if entry.get("directory"):
                     directory_mods.append(entry_name)
@@ -328,6 +349,7 @@ def _get_modules_http(self, url):
         return result
 
     def _get_modules_http_dir_mods(self, auth, directory_mods, result, url):
+        # pylint: disable=too-many-locals
         """
         #TODO describe what this does
 
@@ -337,21 +359,27 @@ def _get_modules_http_dir_mods(self, auth, directory_mods, result, url):
         :param url: URL of the device.
         """
         for dm in directory_mods:
-            if not str(urlparse(dm).scheme).lower() in ("http", "https"):
+            if str(urlparse(dm).scheme).lower() not in ("http", "https"):
                 dm_url = url + dm + "/"
             else:
                 dm_url = dm
-            with requests.get(dm_url, auth=auth, headers={"Accept": "application/json"}) as r:
+
+            print(f"dm_url: {dm_url}")
+            with self.session.get(
+                dm_url, auth=auth, headers={"Accept": "application/json"}, timeout=10
+            ) as r:
                 r.raise_for_status()
                 mpy = False
-                for entry in r.json():
+
+                for entry in r.json()["files"]:
                     entry_name = entry.get("name")
                     if not entry.get("directory") and (
                         entry_name.endswith(".py") or entry_name.endswith(".mpy")
                     ):
                         if entry_name.endswith(".mpy"):
                             mpy = True
-                        with requests.get(dm_url + entry_name, auth=auth) as rr:
+
+                        with self.session.get(dm_url + entry_name, auth=auth) as rr:
                             rr.raise_for_status()
                             idx = entry_name.rfind(".")
                             with tempfile.NamedTemporaryFile(
@@ -381,11 +409,11 @@ def _get_modules_http_single_mods(self, auth, result, single_file_mods, url):
         :param url: URL of the device.
         """
         for sfm in single_file_mods:
-            if not str(urlparse(sfm).scheme).lower() in ("http", "https"):
+            if str(urlparse(sfm).scheme).lower() not in ("http", "https"):
                 sfm_url = url + sfm
             else:
                 sfm_url = sfm
-            with requests.get(sfm_url, auth=auth) as r:
+            with self.session.get(sfm_url, auth=auth) as r:
                 r.raise_for_status()
                 idx = sfm.rfind(".")
                 with tempfile.NamedTemporaryFile(
@@ -401,7 +429,7 @@ def _get_modules_http_single_mods(self, auth, result, single_file_mods, url):
     def _create_library_directory(self, device_path, library_path):
         url = urlparse(device_path)
         auth = HTTPBasicAuth("", url.password)
-        with requests.put(library_path, auth=auth) as r:
+        with self.session.put(library_path, auth=auth) as r:
             r.raise_for_status()
 
     def _install_module_mpy(self, bundle, metadata):
@@ -455,7 +483,7 @@ def get_auto_file_path(self, auto_file_path):
         """
         url = auto_file_path
         auth = HTTPBasicAuth("", self.password)
-        with requests.get(url, auth=auth) as r:
+        with self.session.get(url, auth=auth) as r:
             r.raise_for_status()
             with open(LOCAL_CODE_PY_COPY, "w", encoding="utf-8") as f:
                 f.write(r.text)
@@ -468,7 +496,7 @@ def uninstall(self, device_path, module_path):
         print(f"Uninstalling {module_path}")
         url = urlparse(device_path)
         auth = HTTPBasicAuth("", url.password)
-        with requests.delete(module_path, auth=auth) as r:
+        with self.session.delete(module_path, auth=auth) as r:
             r.raise_for_status()
 
     def update(self, module):
@@ -491,15 +519,19 @@ def _update_http(self, module):
             # Delete the directory (recursive) first.
             url = urlparse(module.path)
             auth = HTTPBasicAuth("", url.password)
-            with requests.delete(module.path, auth=auth) as r:
+            with self.session.delete(module.path, auth=auth) as r:
                 r.raise_for_status()
             self.install_dir_http(module.bundle_path)
 
     def get_file_path(self, filename):
         """
         retuns the full path on the device to a given file name.
         """
-        return urljoin( urljoin(self.device_location, "fs/", allow_fragments=False), filename, allow_fragments=False)
+        return urljoin(
+            urljoin(self.device_location, "fs/", allow_fragments=False),
+            filename,
+            allow_fragments=False,
+        )
 
     def is_device_present(self):
         """
@@ -521,6 +553,7 @@ def get_device_versions(self):
         """
         return self.get_modules(urljoin(self.device_location, self.LIB_DIR_PATH))
 
+
 class DiskBackend(Backend):
     """
     Backend for interacting with a device via USB Workflow