From dbb8374638ae092bf806051cec2d52e87c00c0f8 Mon Sep 17 00:00:00 2001 From: foamyguy Date: Mon, 26 Feb 2024 17:30:04 -0600 Subject: [PATCH] fix for json response files key. Use requests Session with retries in WebBackend. timeout on a requests call. code format --- circup/__init__.py | 21 ++++++--- circup/backends.py | 107 +++++++++++++++++++++++++++++---------------- 2 files changed, 85 insertions(+), 43 deletions(-) diff --git a/circup/__init__.py b/circup/__init__.py index 9d2cece..36d39a1 100644 --- a/circup/__init__.py +++ b/circup/__init__.py @@ -264,8 +264,12 @@ def __init__( """ self.name = name self.backend = backend - self.path = urljoin(backend.library_path,name, allow_fragments=False) if isinstance(backend,WebBackend) else os.path.join(backend.library_path, name) - url = urlparse(self.path,allow_fragments=False) + self.path = ( + urljoin(backend.library_path, name, allow_fragments=False) + if isinstance(backend, WebBackend) + else os.path.join(backend.library_path, name) + ) + url = urlparse(self.path, allow_fragments=False) if str(url.scheme).lower() in ("http", "https"): if url.path.endswith(".py") or url.path.endswith(".mpy"): self.file = os.path.basename(url.path) @@ -274,7 +278,9 @@ def __init__( ) else: self.file = None - self.name = os.path.basename(url.path if url.path[:-1]=='/' else url.path[:-1]) + self.name = os.path.basename( + url.path if url.path[:-1] == "/" else url.path[:-1] + ) else: if os.path.isfile(self.path): # Single file module. @@ -608,7 +614,9 @@ def find_modules(backend, bundles_list): if not path.endswith(os.sep) else path[:-1].split(os.sep)[-1] + os.sep ) - module_name = name if not path.find(os.sep) else module_name # should probably check for os.sep and use previous version if found + module_name = ( + name if not path.find(os.sep) else module_name + ) # should probably check for os.sep and use previous version if found m = Module( module_name, backend, @@ -1224,7 +1232,9 @@ def install(ctx, modules, pyext, requirement, auto, auto_file): # pragma: no co auto_file = "code.py" print(f"Auto file: {auto_file}") # pass a local file with "./" or "../" - is_relative = not isinstance(ctx.obj["backend"], WebBackend) or auto_file.split(os.sep)[0] in [os.path.curdir, os.path.pardir] + is_relative = not isinstance(ctx.obj["backend"], WebBackend) or auto_file.split( + os.sep + )[0] in [os.path.curdir, os.path.pardir] if not os.path.isabs(auto_file) and not is_relative: auto_file = ctx.obj["backend"].get_file_path(auto_file or "code.py") @@ -1301,7 +1311,6 @@ def uninstall(ctx, module): # pragma: no cover continue - # pylint: disable=too-many-branches diff --git a/circup/backends.py b/circup/backends.py index afd1d21..59a81b1 100644 --- a/circup/backends.py +++ b/circup/backends.py @@ -13,6 +13,7 @@ from urllib.parse import urlparse, urljoin import click import requests +from requests.adapters import HTTPAdapter from requests.auth import HTTPBasicAuth @@ -116,7 +117,11 @@ def install_module( click.echo("'{}' is already installed.".format(name)) return - library_path = os.path.join(device_path, self.LIB_DIR_PATH) if not isinstance(self,WebBackend) else urljoin(device_path,self.LIB_DIR_PATH) + library_path = ( + os.path.join(device_path, self.LIB_DIR_PATH) + if not isinstance(self, WebBackend) + else urljoin(device_path, self.LIB_DIR_PATH) + ) # Create the library directory first. self._create_library_directory(device_path, library_path) @@ -206,6 +211,8 @@ def __init__(self, host, password, logger): self.password = password self.device_location = f"http://:{self.password}@{self.host}" + self.session = requests.Session() + self.session.mount(self.device_location, HTTPAdapter(max_retries=5)) self.library_path = self.device_location + "/" + self.LIB_DIR_PATH def install_file_http(self, source): @@ -215,19 +222,14 @@ def install_file_http(self, source): """ file_name = source.split(os.path.sep) file_name = file_name[-2] if file_name[-1] == "" else file_name[-1] - target = ( - self.device_location - + "/" - + self.LIB_DIR_PATH - + file_name - ) + target = self.device_location + "/" + self.LIB_DIR_PATH + file_name auth = HTTPBasicAuth("", self.password) print(f"target: {target}") print(f"source: {source}") with open(source, "rb") as fp: - r = requests.put(target, fp.read(), auth=auth) + r = self.session.put(target, fp.read(), auth=auth) r.raise_for_status() def install_dir_http(self, source): @@ -237,20 +239,15 @@ def install_dir_http(self, source): """ mod_name = source.split(os.path.sep) mod_name = mod_name[-2] if mod_name[-1] == "" else mod_name[-1] - target = ( - self.device_location - + "/" - + self.LIB_DIR_PATH - + mod_name - ) - target = target + "/" if target[:-1]!="/" else target + target = self.device_location + "/" + self.LIB_DIR_PATH + mod_name + target = target + "/" if target[:-1] != "/" else target url = urlparse(target) auth = HTTPBasicAuth("", url.password) print(f"target: {target}") print(f"source: {source}") # Create the top level directory. - with requests.put(target, auth=auth) as r: + with self.session.put(target, auth=auth) as r: print(f"resp {r.content}") r.raise_for_status() @@ -260,18 +257,36 @@ def install_dir_http(self, source): if rel_path == ".": rel_path = "" for name in dirs: - path_to_create = urljoin( urljoin(target , rel_path + "/", allow_fragments=False) , name, allow_fragments=False) if rel_path != "" else urljoin(target , name, allow_fragments=False) - path_to_create = path_to_create + "/" if path_to_create[:-1]!="/" else path_to_create + path_to_create = ( + urljoin( + urljoin(target, rel_path + "/", allow_fragments=False), + name, + allow_fragments=False, + ) + if rel_path != "" + else urljoin(target, name, allow_fragments=False) + ) + path_to_create = ( + path_to_create + "/" + if path_to_create[:-1] != "/" + else path_to_create + ) print(f"dir_path_to_create: {path_to_create}") - with requests.put(path_to_create, auth=auth) as r: + with self.session.put(path_to_create, auth=auth) as r: r.raise_for_status() for name in files: with open(os.path.join(root, name), "rb") as fp: - path_to_create = urljoin( urljoin(target , rel_path + "/", allow_fragments=False) , name, allow_fragments=False) if rel_path != "" else urljoin(target , name, allow_fragments=False) + path_to_create = ( + urljoin( + urljoin(target, rel_path + "/", allow_fragments=False), + name, + allow_fragments=False, + ) + if rel_path != "" + else urljoin(target, name, allow_fragments=False) + ) print(f"file_path_to_create: {path_to_create}") - with requests.put( - path_to_create, fp.read(), auth=auth - ) as r: + with self.session.put(path_to_create, fp.read(), auth=auth) as r: r.raise_for_status() def get_circuitpython_version(self): @@ -283,7 +298,7 @@ def get_circuitpython_version(self): :return: A tuple with the version string for CircuitPython and the board ID string. """ # pylint: disable=arguments-renamed - with requests.get(self.device_location + "/cp/version.json") as r: + with self.session.get(self.device_location + "/cp/version.json") as r: # pylint: disable=no-member if r.status_code != requests.codes.ok: click.secho( @@ -296,6 +311,7 @@ def get_circuitpython_version(self): return ver_json.get("version"), ver_json.get("board_id") def _get_modules(self, device_lib_path): + print(f"device_lib_path {device_lib_path}") return self._get_modules_http(device_lib_path) def _get_modules_http(self, url): @@ -309,12 +325,17 @@ def _get_modules_http(self, url): result = {} u = urlparse(url) auth = HTTPBasicAuth("", u.password) - with requests.get(url, auth=auth, headers={"Accept": "application/json"}) as r: + with self.session.get( + url, auth=auth, headers={"Accept": "application/json"} + ) as r: r.raise_for_status() directory_mods = [] single_file_mods = [] - for entry in r.json(): + + for entry in r.json()["files"]: + # print(f"type: {type(entry)}") + # print(f"val: {entry}") entry_name = entry.get("name") if entry.get("directory"): directory_mods.append(entry_name) @@ -328,6 +349,7 @@ def _get_modules_http(self, url): return result def _get_modules_http_dir_mods(self, auth, directory_mods, result, url): + # pylint: disable=too-many-locals """ #TODO describe what this does @@ -337,21 +359,27 @@ def _get_modules_http_dir_mods(self, auth, directory_mods, result, url): :param url: URL of the device. """ for dm in directory_mods: - if not str(urlparse(dm).scheme).lower() in ("http", "https"): + if str(urlparse(dm).scheme).lower() not in ("http", "https"): dm_url = url + dm + "/" else: dm_url = dm - with requests.get(dm_url, auth=auth, headers={"Accept": "application/json"}) as r: + + print(f"dm_url: {dm_url}") + with self.session.get( + dm_url, auth=auth, headers={"Accept": "application/json"}, timeout=10 + ) as r: r.raise_for_status() mpy = False - for entry in r.json(): + + for entry in r.json()["files"]: entry_name = entry.get("name") if not entry.get("directory") and ( entry_name.endswith(".py") or entry_name.endswith(".mpy") ): if entry_name.endswith(".mpy"): mpy = True - with requests.get(dm_url + entry_name, auth=auth) as rr: + + with self.session.get(dm_url + entry_name, auth=auth) as rr: rr.raise_for_status() idx = entry_name.rfind(".") with tempfile.NamedTemporaryFile( @@ -381,11 +409,11 @@ def _get_modules_http_single_mods(self, auth, result, single_file_mods, url): :param url: URL of the device. """ for sfm in single_file_mods: - if not str(urlparse(sfm).scheme).lower() in ("http", "https"): + if str(urlparse(sfm).scheme).lower() not in ("http", "https"): sfm_url = url + sfm else: sfm_url = sfm - with requests.get(sfm_url, auth=auth) as r: + with self.session.get(sfm_url, auth=auth) as r: r.raise_for_status() idx = sfm.rfind(".") with tempfile.NamedTemporaryFile( @@ -401,7 +429,7 @@ def _get_modules_http_single_mods(self, auth, result, single_file_mods, url): def _create_library_directory(self, device_path, library_path): url = urlparse(device_path) auth = HTTPBasicAuth("", url.password) - with requests.put(library_path, auth=auth) as r: + with self.session.put(library_path, auth=auth) as r: r.raise_for_status() def _install_module_mpy(self, bundle, metadata): @@ -455,7 +483,7 @@ def get_auto_file_path(self, auto_file_path): """ url = auto_file_path auth = HTTPBasicAuth("", self.password) - with requests.get(url, auth=auth) as r: + with self.session.get(url, auth=auth) as r: r.raise_for_status() with open(LOCAL_CODE_PY_COPY, "w", encoding="utf-8") as f: f.write(r.text) @@ -468,7 +496,7 @@ def uninstall(self, device_path, module_path): print(f"Uninstalling {module_path}") url = urlparse(device_path) auth = HTTPBasicAuth("", url.password) - with requests.delete(module_path, auth=auth) as r: + with self.session.delete(module_path, auth=auth) as r: r.raise_for_status() def update(self, module): @@ -491,7 +519,7 @@ def _update_http(self, module): # Delete the directory (recursive) first. url = urlparse(module.path) auth = HTTPBasicAuth("", url.password) - with requests.delete(module.path, auth=auth) as r: + with self.session.delete(module.path, auth=auth) as r: r.raise_for_status() self.install_dir_http(module.bundle_path) @@ -499,7 +527,11 @@ def get_file_path(self, filename): """ retuns the full path on the device to a given file name. """ - return urljoin( urljoin(self.device_location, "fs/", allow_fragments=False), filename, allow_fragments=False) + return urljoin( + urljoin(self.device_location, "fs/", allow_fragments=False), + filename, + allow_fragments=False, + ) def is_device_present(self): """ @@ -521,6 +553,7 @@ def get_device_versions(self): """ return self.get_modules(urljoin(self.device_location, self.LIB_DIR_PATH)) + class DiskBackend(Backend): """ Backend for interacting with a device via USB Workflow