From 8b413ffe1604c6dcd808c1fdbd5f2240bf0a0dc5 Mon Sep 17 00:00:00 2001 From: Vladimir Kotal Date: Wed, 22 Feb 2023 20:25:56 +0100 Subject: [PATCH 01/63] add support for web workflow fixes #156 --- .isort.cfg | 5 + circup/__init__.py | 497 +++++++++++++++++++++++++++++++++++-------- tests/test_circup.py | 4 +- 3 files changed, 420 insertions(+), 86 deletions(-) create mode 100644 .isort.cfg diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 0000000..4c0cb09 --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: 2023 Vladimír Kotal +# +# SPDX-License-Identifier: Unlicense +[settings] +profile = black diff --git a/circup/__init__.py b/circup/__init__.py index 2854983..441ebd8 100644 --- a/circup/__init__.py +++ b/circup/__init__.py @@ -13,18 +13,21 @@ import os import re import shutil -from subprocess import check_output +import socket import sys +import tempfile import zipfile +from subprocess import check_output +from urllib.parse import urlparse import appdirs import click import findimports import pkg_resources import requests -from semver import VersionInfo import update_checker - +from requests.auth import HTTPBasicAuth +from semver import VersionInfo # Useful constants. #: Flag to indicate if the command is being run in verbose mode. @@ -237,7 +240,7 @@ def __init__( resulting self.file value will be None, and the name will be the basename of the directory path. - :param str path: The path to the module on the connected + :param str path: The path or URL to the module on the connected CIRCUITPYTHON device. :param str repo: The URL of the Git repository for this module. :param str device_version: The semver value for the version on device. @@ -247,14 +250,25 @@ def __init__( :param (str,str) compatibility: Min and max versions of CP compatible with the mpy. """ self.path = path - if os.path.isfile(self.path): - # Single file module. - self.file = os.path.basename(path) - self.name = self.file.replace(".py", "").replace(".mpy", "") + url = urlparse(path) + if url.scheme == "http": + if url.path.endswith(".py") or url.path.endswith(".mpy"): + self.file = os.path.basename(url.path) + self.name = ( + os.path.basename(url.path).replace(".py", "").replace(".mpy", "") + ) + else: + self.file = None + self.name = os.path.basename(url.path[:-1]) else: - # Directory based module. - self.file = None - self.name = os.path.basename(os.path.dirname(self.path)) + if os.path.isfile(self.path): + # Single file module. + self.file = os.path.basename(path) + self.name = self.file.replace(".py", "").replace(".mpy", "") + else: + # Directory based module. + self.file = None + self.name = os.path.basename(os.path.dirname(self.path)) self.repo = repo self.device_version = device_version self.bundle_version = bundle_version @@ -377,6 +391,32 @@ def update(self): The caller is expected to handle any exceptions raised. """ + url = urlparse(self.path) + if url.scheme == "http": + self._update_http() + else: + self._update_file() + + def _update_http(self): + """ + Update the module using web workflow. + """ + if self.file: + # Copy the file (will overwrite). + install_file_http(self.bundle_path, self.path) + else: + # Delete the directory (recursive) first. + url = urlparse(self.path) + auth = HTTPBasicAuth("", url.password) + r = requests.delete(self.path, auth=auth) + r.raise_for_status() + + install_dir_http(self.bundle_path, self.path) + + def _update_file(self): + """ + Update the module using file system. + """ if os.path.isdir(self.path): # Delete and copy the directory. shutil.rmtree(self.path, ignore_errors=True) @@ -408,6 +448,45 @@ def __repr__(self): ) +def install_file_http(source, target): + """ + Install file to device using web workflow. + :param source source file. + :param target destination URL. Should have password embedded. + """ + url = urlparse(target) + auth = HTTPBasicAuth("", url.password) + + with open(source, "rb") as fp: + r = requests.put(target, fp.read(), auth=auth) + r.raise_for_status() + + +def install_dir_http(source, target): + """ + Install directory to device using web workflow. + :param source source directory. + :param target destination URL. Should have password embedded. + """ + url = urlparse(target) + auth = HTTPBasicAuth("", url.password) + + # Create the top level directory. + r = requests.put(target, auth=auth) + r.raise_for_status() + + # Traverse the directory structure and create the directories/files. + for root, dirs, files in os.walk(source): + rel_path = os.path.relpath(root, source) + for name in files: + with open(os.path.join(root, name), "rb") as fp: + r = requests.put(target + rel_path + "/" + name, fp.read(), auth=auth) + r.raise_for_status() + for name in dirs: + r = requests.put(target + rel_path + "/" + name, auth=auth) + r.raise_for_status() + + def clean_library_name(assumed_library_name): """ Most CP repos and library names are look like this: @@ -524,8 +603,6 @@ def extract_metadata(path): dunder_key_val = r"""(__\w+__)(?:\s*:\s*\w+)?\s*=\s*(?:['"]|\(\s)(.+)['"]""" for match in re.findall(dunder_key_val, content): result[match[0]] = str(match[1]) - if result: - logger.info("Extracted metadata: %s", result) elif path.endswith(".mpy"): result["mpy"] = True with open(path, "rb") as mpy_file: @@ -564,6 +641,9 @@ def extract_metadata(path): else: # not a valid MPY file result["__version__"] = BAD_FILE_FORMAT + + if result: + logger.info("Extracted metadata: %s", result) return result @@ -635,20 +715,20 @@ def get_volume_name(disk_name): return device_dir -def find_modules(device_path, bundles_list): +def find_modules(device_url, bundles_list): """ Extracts metadata from the connected device and available bundles and returns this as a list of Module instances representing the modules on the device. - :param str device_path: The path to the connected board. + :param str device_url: The URL to the board. :param Bundle bundles_list: List of supported bundles as Bundle objects. :return: A list of Module instances describing the current state of the modules on the connected device. """ # pylint: disable=broad-except,too-many-locals try: - device_modules = get_device_versions(device_path) + device_modules = get_device_versions(device_url) bundle_modules = get_bundle_versions(bundles_list) result = [] for name, device_metadata in device_modules.items(): @@ -661,17 +741,16 @@ def find_modules(device_path, bundles_list): bundle_version = bundle_metadata.get("__version__") mpy = device_metadata["mpy"] compatibility = device_metadata.get("compatibility", (None, None)) - result.append( - Module( - path, - repo, - device_version, - bundle_version, - mpy, - bundle, - compatibility, - ) + m = Module( + path, + repo, + device_version, + bundle_version, + mpy, + bundle, + compatibility, ) + result.append(m) return result except Exception as ex: # If it's not possible to get the device and bundle metadata, bail out @@ -734,7 +813,7 @@ def get_bundle_versions(bundles_list, avoid_download=False): if not avoid_download or not os.path.isdir(bundle.lib_dir("py")): ensure_latest_bundle(bundle) path = bundle.lib_dir("py") - path_modules = get_modules(path) + path_modules = _get_modules_file(path) for name, module in path_modules.items(): module["bundle"] = bundle if name not in all_the_modules: # here we decide the order of priority @@ -792,7 +871,44 @@ def get_bundles_list(): return bundles_list -def get_circuitpython_version(device_path): +def get_circuitpython_version(device_url): + """ + Returns the version number of CircuitPython running on the board connected + via ``device_path``, along with the board ID. + + :param str device_url: device URL. Can be either file or http based. + :return: A tuple with the version string for CircuitPython and the board ID string. + """ + url = urlparse(device_url) + if url.scheme == "http": + return _get_circuitpython_version_http(device_url) + if url.scheme == "": + return _get_circuitpython_version_file(url.path) + + click.secho(f"Not supported URL scheme: {url.scheme}", fg="red") + sys.exit(1) + + +def _get_circuitpython_version_http(url): + """ + Returns the version number of CircuitPython running on the board connected + via ``device_path``, along with the board ID. This is obtained using + RESTful API from the /cp/version.json URL. + + :param str url: board URL. + :return: A tuple with the version string for CircuitPython and the board ID string. + """ + r = requests.get(url + "/cp/version.json") + # pylint: disable=no-member + if r.status_code != requests.codes.ok: + click.secho(f" Unable to get version from {url}: {r.status_code}", fg="red") + sys.exit(1) + # pylint: enable=no-member + ver_json = r.json() + return ver_json.get("version"), ver_json.get("board_id") + + +def _get_circuitpython_version_file(device_path): """ Returns the version number of CircuitPython running on the board connected via ``device_path``, along with the board ID. This is obtained from the @@ -808,6 +924,7 @@ def get_circuitpython_version(device_path): :param str device_path: The path to the connected board. :return: A tuple with the version string for CircuitPython and the board ID string. """ + try: with open( os.path.join(device_path, "boot_out.txt"), "r", encoding="utf-8" @@ -826,7 +943,8 @@ def get_circuitpython_version(device_path): ) logger.error("boot_out.txt not found.") sys.exit(1) - return (circuit_python, board_id) + + return circuit_python, board_id def get_circup_version(): @@ -907,15 +1025,34 @@ def get_dependencies(*requested_libraries, mod_names, to_install=()): ) -def get_device_versions(device_path): +def get_device_versions(device_url): """ Returns a dictionary of metadata from modules on the connected device. - :param str device_path: Path to the device volume. + :param str device_url: URL for the device. :return: A dictionary of metadata about the modules available on the connected device. """ - return get_modules(os.path.join(device_path, "lib")) + url = urlparse(device_url) + if url.scheme == "http": + return get_modules(device_url + "/fs/lib/") + + return get_modules(os.path.join(url.path, "lib")) + + +def get_modules(device_url): + """ + Get a dictionary containing metadata about all the Python modules found in + the referenced path. + + :param str device_url: URL to be used to find modules. + :return: A dictionary containing metadata about the found modules. + """ + url = urlparse(device_url) + if url.scheme == "http": + return _get_modules_http(device_url) + + return _get_modules_file(device_url) def get_latest_release_from_url(url): @@ -936,10 +1073,103 @@ def get_latest_release_from_url(url): return tag -def get_modules(path): +def _get_modules_http(url): + """ + Get a dictionary containing metadata about all the Python modules found using + the referenced URL. + + :param str url: URL for the modules. + :return: A dictionary containing metadata about the found modules. + """ + result = {} + u = urlparse(url) + auth = HTTPBasicAuth("", u.password) + r = requests.get(url, auth=auth, headers={"Accept": "application/json"}) + r.raise_for_status() + + directory_mods = [] + single_file_mods = [] + for entry in r.json(): + entry_name = entry.get("name") + if entry.get("directory"): + directory_mods.append(entry_name) + else: + if entry_name.endswith(".py") or entry_name.endswith(".mpy"): + single_file_mods.append(entry_name) + + _get_modules_http_single_mods(auth, result, single_file_mods, url) + _get_modules_http_dir_mods(auth, directory_mods, result, url) + + return result + + +def _get_modules_http_dir_mods(auth, directory_mods, result, url): + """ + :param auth HTTP authentication. + :param directory_mods list of modules. + :param result dictionary for the result. + :param url: URL of the device. + """ + for dm in directory_mods: + dm_url = url + dm + "/" + r = requests.get(dm_url, auth=auth, headers={"Accept": "application/json"}) + r.raise_for_status() + mpy = False + for entry in r.json(): + entry_name = entry.get("name") + if not entry.get("directory") and ( + entry_name.endswith(".py") or entry_name.endswith(".mpy") + ): + if entry_name.endswith(".mpy"): + mpy = True + r = requests.get(dm_url + entry_name, auth=auth) + r.raise_for_status() + idx = entry_name.rfind(".") + with tempfile.NamedTemporaryFile( + prefix=entry_name[:idx] + "-", suffix=entry_name[idx:], delete=False + ) as fp: + fp.write(r.content) + tmp_name = fp.name + metadata = extract_metadata(tmp_name) + os.remove(tmp_name) + if "__version__" in metadata: + metadata["path"] = dm_url + result[dm] = metadata + # break now if any of the submodules has a bad format + if metadata["__version__"] == BAD_FILE_FORMAT: + break + + if result.get(dm) is None: + result[dm] = {"path": dm_url, "mpy": mpy} + + +def _get_modules_http_single_mods(auth, result, single_file_mods, url): + """ + :param auth HTTP authentication. + :param single_file_mods list of modules. + :param result dictionary for the result. + :param url: URL of the device. + """ + for sfm in single_file_mods: + sfm_url = url + sfm + r = requests.get(sfm_url, auth=auth) + r.raise_for_status() + idx = sfm.rfind(".") + with tempfile.NamedTemporaryFile( + prefix=sfm[:idx] + "-", suffix=sfm[idx:], delete=False + ) as fp: + fp.write(r.content) + tmp_name = fp.name + metadata = extract_metadata(tmp_name) + os.remove(tmp_name) + metadata["path"] = sfm_url + result[sfm[:idx]] = metadata + + +def _get_modules_file(path): """ Get a dictionary containing metadata about all the Python modules found in - the referenced path. + the referenced file system path. :param str path: The directory in which to find modules. :return: A dictionary containing metadata about the found modules. @@ -999,53 +1229,94 @@ def install_module( if not name: click.echo("No module name(s) provided.") elif name in mod_names: - library_path = os.path.join(device_path, "lib") - if not os.path.exists(library_path): # pragma: no cover - os.makedirs(library_path) - metadata = mod_names[name] - bundle = metadata["bundle"] # Grab device modules to check if module already installed if name in device_modules: click.echo("'{}' is already installed.".format(name)) return + + # Create the library directory first. + url = urlparse(device_path) + if url.scheme == "http": + library_path = device_path + "/fs/lib/" + auth = HTTPBasicAuth("", url.password) + r = requests.put(library_path, auth=auth) + r.raise_for_status() + else: + library_path = os.path.join(device_path, "lib") + if not os.path.exists(library_path): # pragma: no cover + os.makedirs(library_path) + + metadata = mod_names[name] + bundle = metadata["bundle"] if pyext: # Use Python source for module. - source_path = metadata["path"] # Path to Python source version. - if os.path.isdir(source_path): - target = os.path.basename(os.path.dirname(source_path)) - target_path = os.path.join(library_path, target) - # Copy the directory. - shutil.copytree(source_path, target_path) - else: - target = os.path.basename(source_path) - target_path = os.path.join(library_path, target) - # Copy file. - shutil.copyfile(source_path, target_path) + _install_module_py(library_path, metadata) else: # Use pre-compiled mpy modules. - module_name = os.path.basename(metadata["path"]).replace(".py", ".mpy") - if not module_name: - # Must be a directory based module. - module_name = os.path.basename(os.path.dirname(metadata["path"])) - major_version = CPY_VERSION.split(".")[0] - bundle_platform = "{}mpy".format(major_version) - bundle_path = os.path.join(bundle.lib_dir(bundle_platform), module_name) - if os.path.isdir(bundle_path): - target_path = os.path.join(library_path, module_name) - # Copy the directory. - shutil.copytree(bundle_path, target_path) - elif os.path.isfile(bundle_path): - target = os.path.basename(bundle_path) - target_path = os.path.join(library_path, target) - # Copy file. - shutil.copyfile(bundle_path, target_path) - else: - raise IOError("Cannot find compiled version of module.") + _install_module_mpy(bundle, library_path, metadata) click.echo("Installed '{}'.".format(name)) else: click.echo("Unknown module named, '{}'.".format(name)) +def _install_module_mpy(bundle, library_path, metadata): + """ + :param bundle library bundle. + :param library_path library path + :param metadata dictionary. + """ + url = urlparse(library_path) + module_name = os.path.basename(metadata["path"]).replace(".py", ".mpy") + if not module_name: + # Must be a directory based module. + module_name = os.path.basename(os.path.dirname(metadata["path"])) + major_version = CPY_VERSION.split(".")[0] + bundle_platform = "{}mpy".format(major_version) + bundle_path = os.path.join(bundle.lib_dir(bundle_platform), module_name) + if os.path.isdir(bundle_path): + if url.scheme == "http": + install_dir_http(bundle_path, library_path + module_name) + else: + target_path = os.path.join(library_path, module_name) + # Copy the directory. + shutil.copytree(bundle_path, target_path) + elif os.path.isfile(bundle_path): + target = os.path.basename(bundle_path) + if url.scheme == "http": + install_file_http(bundle_path, library_path + target) + else: + target_path = os.path.join(library_path, target) + # Copy file. + shutil.copyfile(bundle_path, target_path) + else: + raise IOError("Cannot find compiled version of module.") + + +def _install_module_py(library_path, metadata): + """ + :param library_path library path + :param metadata dictionary. + """ + url = urlparse(library_path) + source_path = metadata["path"] # Path to Python source version. + if os.path.isdir(source_path): + target = os.path.basename(os.path.dirname(source_path)) + if url.scheme == "http": + install_dir_http(source_path, library_path + target) + else: + target_path = os.path.join(library_path, target) + # Copy the directory. + shutil.copytree(source_path, target_path) + else: + target = os.path.basename(source_path) + if url.scheme == "http": + install_file_http(source_path, library_path + target) + else: + target_path = os.path.join(library_path, target) + # Copy file. + shutil.copyfile(source_path, target_path) + + # pylint: enable=too-many-locals,too-many-branches @@ -1148,12 +1419,19 @@ def tags_data_save_tag(key, tag): type=click.Path(exists=True, file_okay=False), help="Path to CircuitPython directory. Overrides automatic path detection.", ) +@click.option( + "--host", + help="Hostname or IP address of a device. Overrides automatic path detection.", +) +@click.option( + "--password", help="Password to use for authentication when --host is used." +) @click.version_option( prog_name="CircUp", message="%(prog)s, A CircuitPython module updater. Version %(version)s", ) @click.pass_context -def main(ctx, verbose, path): # pragma: no cover +def main(ctx, verbose, path, host, password): # pragma: no cover """ A tool to manage and update libraries on a CircuitPython device. """ @@ -1178,10 +1456,8 @@ def main(ctx, verbose, path): # pragma: no cover # stop early if the command is boardless if ctx.invoked_subcommand in BOARDLESS_COMMANDS: return - if path: - device_path = path - else: - device_path = find_device() + + device_path = get_device_path(host, password, path) ctx.obj["DEVICE_PATH"] = device_path latest_version = get_latest_release_from_url( "https://github.com/adafruit/circuitpython/releases/latest" @@ -1215,6 +1491,37 @@ def main(ctx, verbose, path): # pragma: no cover logger.warning(ex) +def get_device_path(host, password, path): + """ + :param host Hostname or IP address. + :param password REST API password. + :param path File system path. + :return device URL or None if the device cannot be found. + """ + if path: + device_path = "file:///" + path + elif host: + if password is None: + click.secho("--host needs --password", fg="red") + sys.exit(1) + + # pylint: disable=no-member + # verify hostname/address + try: + socket.getaddrinfo(host, 80, proto=socket.IPPROTO_TCP) + except socket.gaierror: + click.secho("Invalid host: {}".format(host), fg="red") + sys.exit(1) + # pylint: enable=no-member + device_path = f"http://:{password}@" + host + else: + device_path = find_device() + if device_path is not None: + device_path = "file:///" + device_path + + return device_path + + @main.command() @click.option("-r", "--requirement", is_flag=True) @click.pass_context @@ -1368,31 +1675,53 @@ def uninstall(ctx, module): # pragma: no cover can be uninstalled at once by providing more than one module name, each separated by a space. """ + device_path = ctx.obj["DEVICE_PATH"] for name in module: - device_modules = get_device_versions(ctx.obj["DEVICE_PATH"]) + device_modules = get_device_versions(device_path) name = name.lower() mod_names = {} for module_item, metadata in device_modules.items(): mod_names[module_item.replace(".py", "").lower()] = metadata if name in mod_names: - library_path = os.path.join(ctx.obj["DEVICE_PATH"], "lib") metadata = mod_names[name] module_path = metadata["path"] - if os.path.isdir(module_path): - target = os.path.basename(os.path.dirname(module_path)) - target_path = os.path.join(library_path, target) - # Remove the directory. - shutil.rmtree(target_path) + url = urlparse(device_path) + if url.scheme == "http": + _uninstall_http(device_path, module_path) else: - target = os.path.basename(module_path) - target_path = os.path.join(library_path, target) - # Remove file - os.remove(target_path) + _uninstall_file(device_path, module_path) click.echo("Uninstalled '{}'.".format(name)) else: click.echo("Module '{}' not found on device.".format(name)) +def _uninstall_http(device_path, module_path): + """ + Uninstall given module on device using REST API. + """ + url = urlparse(device_path) + auth = HTTPBasicAuth("", url.password) + r = requests.delete(module_path, auth=auth) + r.raise_for_status() + + +def _uninstall_file(device_path, module_path): + """ + Uninstall module using local file system. + """ + library_path = os.path.join(device_path, "lib") + if os.path.isdir(module_path): + target = os.path.basename(os.path.dirname(module_path)) + target_path = os.path.join(library_path, target) + # Remove the directory. + shutil.rmtree(target_path) + else: + target = os.path.basename(module_path) + target_path = os.path.join(library_path, target) + # Remove file + os.remove(target_path) + + # pylint: disable=too-many-branches diff --git a/tests/test_circup.py b/tests/test_circup.py index 1faa69b..64331f3 100644 --- a/tests/test_circup.py +++ b/tests/test_circup.py @@ -647,7 +647,7 @@ def test_get_bundle_versions(): Ensure ensure_latest_bundle is called even if lib_dir exists. """ with mock.patch("circup.ensure_latest_bundle") as mock_elb, mock.patch( - "circup.get_modules", return_value={"ok": {"name": "ok"}} + "circup._get_modules_file", return_value={"ok": {"name": "ok"}} ) as mock_gm, mock.patch("circup.CPY_VERSION", "4.1.2"), mock.patch( "circup.Bundle.lib_dir", return_value="foo/bar/lib" ), mock.patch( @@ -668,7 +668,7 @@ def test_get_bundle_versions_avoid_download(): Testing both cases: lib_dir exists and lib_dir doesn't exists. """ with mock.patch("circup.ensure_latest_bundle") as mock_elb, mock.patch( - "circup.get_modules", return_value={"ok": {"name": "ok"}} + "circup._get_modules_file", return_value={"ok": {"name": "ok"}} ) as mock_gm, mock.patch("circup.CPY_VERSION", "4.1.2"), mock.patch( "circup.Bundle.lib_dir", return_value="foo/bar/lib" ): From 151d3585ff2e0ae1acab76dd25cf571c5d273d56 Mon Sep 17 00:00:00 2001 From: Vladimir Kotal Date: Wed, 22 Feb 2023 22:10:42 +0100 Subject: [PATCH 02/63] fix install of multi-file packages --- circup/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/circup/__init__.py b/circup/__init__.py index 441ebd8..c272f0a 100644 --- a/circup/__init__.py +++ b/circup/__init__.py @@ -472,12 +472,14 @@ def install_dir_http(source, target): auth = HTTPBasicAuth("", url.password) # Create the top level directory. - r = requests.put(target, auth=auth) + r = requests.put(target + "/", auth=auth) r.raise_for_status() # Traverse the directory structure and create the directories/files. for root, dirs, files in os.walk(source): rel_path = os.path.relpath(root, source) + if rel_path == ".": + rel_path = "" for name in files: with open(os.path.join(root, name), "rb") as fp: r = requests.put(target + rel_path + "/" + name, fp.read(), auth=auth) From 63dcbb994435aee4cfb0b45f16fd5ecf4fee85c3 Mon Sep 17 00:00:00 2001 From: foamyguy Date: Sat, 28 Oct 2023 12:10:03 -0500 Subject: [PATCH 03/63] disable too-many-args for main --- circup/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/circup/__init__.py b/circup/__init__.py index faf558b..89c90c6 100644 --- a/circup/__init__.py +++ b/circup/__init__.py @@ -1454,7 +1454,8 @@ def tags_data_save_tag(key, tag): message="%(prog)s, A CircuitPython module updater. Version %(version)s", ) @click.pass_context -def main(ctx, verbose, path, host, password, board_id, cpy_version): # pragma: no cover +def main(ctx, verbose, path, host, password, board_id, cpy_version): # pragma: no cover + # pylint: disable=too-many-arguments """ A tool to manage and update libraries on a CircuitPython device. """ From 898750524172cdf8df226e2e2ccef225295a4ae3 Mon Sep 17 00:00:00 2001 From: foamyguy Date: Fri, 10 Nov 2023 17:51:40 -0600 Subject: [PATCH 04/63] implementing --auto for webworkflow --- circup/__init__.py | 28 ++++++++++++++++++++++++---- tests/test_circup.py | 2 +- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/circup/__init__.py b/circup/__init__.py index 89c90c6..fd3a962 100644 --- a/circup/__init__.py +++ b/circup/__init__.py @@ -48,6 +48,9 @@ LOG_DIR = appdirs.user_log_dir(appname="circup", appauthor="adafruit") #: The location of the log file for the utility. LOGFILE = os.path.join(LOG_DIR, "circup.log") +#: The localtion to store a local copy of code.py for use with --auto and +# web workflow +LOCAL_CODE_PY_COPY = os.path.join(DATA_DIR, "code.tmp.py") #: The libraries (and blank lines) which don't go on devices NOT_MCU_LIBRARIES = [ "", @@ -1323,7 +1326,7 @@ def _install_module_py(library_path, metadata): # pylint: enable=too-many-locals,too-many-branches -def libraries_from_imports(code_py, mod_names): +def libraries_from_imports(ctx, code_py, mod_names): """ Parse the given code.py file and return the imported libraries @@ -1331,6 +1334,17 @@ def libraries_from_imports(code_py, mod_names): :return: sequence of library names """ # pylint: disable=broad-except + if ctx is not None: + using_webworkflow = "host" in ctx.parent.params.keys() + if using_webworkflow: + url = code_py + auth = HTTPBasicAuth("", ctx.parent.params["password"]) + r = requests.get(url, auth=auth) + r.raise_for_status() + with open(LOCAL_CODE_PY_COPY, "w", encoding="utf-8") as f: + f.write(r.text) + + code_py = LOCAL_CODE_PY_COPY try: found_imports = findimports.find_imports(code_py) except Exception as ex: # broad exception because anything could go wrong @@ -1654,6 +1668,7 @@ def install(ctx, modules, pyext, requirement, auto, auto_file): # pragma: no co can be installed at once by providing more than one module name, each separated by a space. """ + using_webworkflow = "host" in ctx.parent.params.keys() # TODO: Ensure there's enough space on the device available_modules = get_bundle_versions(get_bundles_list()) mod_names = {} @@ -1669,11 +1684,16 @@ def install(ctx, modules, pyext, requirement, auto, auto_file): # pragma: no co # pass a local file with "./" or "../" is_relative = auto_file.split(os.sep)[0] in [os.path.curdir, os.path.pardir] if not os.path.isabs(auto_file) and not is_relative: - auto_file = os.path.join(ctx.obj["DEVICE_PATH"], auto_file or "code.py") - if not os.path.isfile(auto_file): + if not using_webworkflow: + auto_file = os.path.join(ctx.obj["DEVICE_PATH"], auto_file or "code.py") + else: + auto_file = os.path.join( + ctx.obj["DEVICE_PATH"], "fs", auto_file or "code.py" + ) + if not os.path.isfile(auto_file) and not using_webworkflow: click.secho(f"Auto file not found: {auto_file}", fg="red") sys.exit(1) - requested_installs = libraries_from_imports(auto_file, mod_names) + requested_installs = libraries_from_imports(ctx, auto_file, mod_names) else: requested_installs = modules requested_installs = sorted(set(requested_installs)) diff --git a/tests/test_circup.py b/tests/test_circup.py index e90f3fd..53e94ec 100644 --- a/tests/test_circup.py +++ b/tests/test_circup.py @@ -1022,7 +1022,7 @@ def test_libraries_from_imports(): "adafruit_touchscreen", ] test_file = str(pathlib.Path(__file__).parent / "import_styles.py") - result = circup.libraries_from_imports(test_file, mod_names) + result = circup.libraries_from_imports(None, test_file, mod_names) print(result) assert result == [ "adafruit_bus_device", From b7eb10b7a296e49a940bc0ce522c5a3387dd021c Mon Sep 17 00:00:00 2001 From: foamyguy Date: Mon, 13 Nov 2023 10:50:45 -0600 Subject: [PATCH 05/63] fix for file protocol --- circup/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circup/__init__.py b/circup/__init__.py index fd3a962..77e1b1f 100644 --- a/circup/__init__.py +++ b/circup/__init__.py @@ -888,7 +888,7 @@ def get_circuitpython_version(device_url): url = urlparse(device_url) if url.scheme == "http": return _get_circuitpython_version_http(device_url) - if url.scheme == "": + if url.scheme == "" or "file": return _get_circuitpython_version_file(url.path) click.secho(f"Not supported URL scheme: {url.scheme}", fg="red") From 225b490295697e02ac6be9a9bb91d70da4878568 Mon Sep 17 00:00:00 2001 From: foamyguy Date: Mon, 13 Nov 2023 11:27:55 -0600 Subject: [PATCH 06/63] fixes for USB workflow and --auto --- circup/__init__.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/circup/__init__.py b/circup/__init__.py index 33cc551..d05ca7f 100644 --- a/circup/__init__.py +++ b/circup/__init__.py @@ -1373,7 +1373,7 @@ def libraries_from_imports(ctx, code_py, mod_names): """ # pylint: disable=broad-except if ctx is not None: - using_webworkflow = "host" in ctx.parent.params.keys() + using_webworkflow = "host" in ctx.parent.params.keys() and ctx.parent.params['host'] is not None if using_webworkflow: url = code_py auth = HTTPBasicAuth("", ctx.parent.params["password"]) @@ -1596,9 +1596,6 @@ def get_device_path(host, password, path): device_path = f"http://:{password}@" + host else: device_path = find_device() - if device_path is not None: - device_path = "file:///" + device_path - return device_path @@ -1706,7 +1703,7 @@ def install(ctx, modules, pyext, requirement, auto, auto_file): # pragma: no co can be installed at once by providing more than one module name, each separated by a space. """ - using_webworkflow = "host" in ctx.parent.params.keys() + using_webworkflow = "host" in ctx.parent.params.keys() and ctx.parent.params['host'] is not None # TODO: Ensure there's enough space on the device available_modules = get_bundle_versions(get_bundles_list()) mod_names = {} From f4618b3e7ca1311fa76ccc92a5277e2f558f557a Mon Sep 17 00:00:00 2001 From: foamyguy Date: Mon, 13 Nov 2023 11:39:28 -0600 Subject: [PATCH 07/63] fix for None ctx.parent. Fix file scheme condition --- circup/__init__.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/circup/__init__.py b/circup/__init__.py index d05ca7f..156942d 100644 --- a/circup/__init__.py +++ b/circup/__init__.py @@ -26,7 +26,6 @@ import pkg_resources import requests import toml -from semver import VersionInfo import update_checker from requests.auth import HTTPBasicAuth from semver import VersionInfo @@ -891,7 +890,7 @@ def get_circuitpython_version(device_url): url = urlparse(device_url) if url.scheme == "http": return _get_circuitpython_version_http(device_url) - if url.scheme == "" or "file": + if url.scheme in ("", "file"): return _get_circuitpython_version_file(url.path) click.secho(f"Not supported URL scheme: {url.scheme}", fg="red") @@ -1373,7 +1372,11 @@ def libraries_from_imports(ctx, code_py, mod_names): """ # pylint: disable=broad-except if ctx is not None: - using_webworkflow = "host" in ctx.parent.params.keys() and ctx.parent.params['host'] is not None + using_webworkflow = ( + ctx.parent is not None + and "host" in ctx.parent.params.keys() + and ctx.parent.params["host"] is not None + ) if using_webworkflow: url = code_py auth = HTTPBasicAuth("", ctx.parent.params["password"]) @@ -1703,7 +1706,11 @@ def install(ctx, modules, pyext, requirement, auto, auto_file): # pragma: no co can be installed at once by providing more than one module name, each separated by a space. """ - using_webworkflow = "host" in ctx.parent.params.keys() and ctx.parent.params['host'] is not None + using_webworkflow = ( + ctx.parent is not None + and "host" in ctx.parent.params.keys() + and ctx.parent.params["host"] is not None + ) # TODO: Ensure there's enough space on the device available_modules = get_bundle_versions(get_bundles_list()) mod_names = {} From 97e60ed4e0c6084e08d0b8984488c62962ea2a82 Mon Sep 17 00:00:00 2001 From: foamyguy Date: Fri, 17 Nov 2023 17:55:50 -0600 Subject: [PATCH 08/63] starting refactor --- circup/__init__.py | 1020 ++++++++++++++++++++++++-------------------- 1 file changed, 564 insertions(+), 456 deletions(-) diff --git a/circup/__init__.py b/circup/__init__.py index a7b91f9..e9470b0 100644 --- a/circup/__init__.py +++ b/circup/__init__.py @@ -452,47 +452,567 @@ def __repr__(self): ) -def install_file_http(source, target): - """ - Install file to device using web workflow. - :param source source file. - :param target destination URL. Should have password embedded. - """ - url = urlparse(target) - auth = HTTPBasicAuth("", url.password) +class WebBackend: + def __init__(self): + pass - with open(source, "rb") as fp: - r = requests.put(target, fp.read(), auth=auth) - r.raise_for_status() + def install_file_http(self, source, target): + """ + Install file to device using web workflow. + :param source source file. + :param target destination URL. Should have password embedded. + """ + url = urlparse(target) + auth = HTTPBasicAuth("", url.password) + with open(source, "rb") as fp: + r = requests.put(target, fp.read(), auth=auth) + r.raise_for_status() -def install_dir_http(source, target): - """ - Install directory to device using web workflow. - :param source source directory. - :param target destination URL. Should have password embedded. - """ - url = urlparse(target) - auth = HTTPBasicAuth("", url.password) - - # Create the top level directory. - r = requests.put(target + "/", auth=auth) - r.raise_for_status() - - # Traverse the directory structure and create the directories/files. - for root, dirs, files in os.walk(source): - rel_path = os.path.relpath(root, source) - if rel_path == ".": - rel_path = "" - for name in files: - with open(os.path.join(root, name), "rb") as fp: - r = requests.put(target + rel_path + "/" + name, fp.read(), auth=auth) + def install_dir_http(self, source, target): + """ + Install directory to device using web workflow. + :param source source directory. + :param target destination URL. Should have password embedded. + """ + url = urlparse(target) + auth = HTTPBasicAuth("", url.password) + + # Create the top level directory. + r = requests.put(target + "/", auth=auth) + r.raise_for_status() + + # Traverse the directory structure and create the directories/files. + for root, dirs, files in os.walk(source): + rel_path = os.path.relpath(root, source) + if rel_path == ".": + rel_path = "" + for name in files: + with open(os.path.join(root, name), "rb") as fp: + r = requests.put(target + rel_path + "/" + name, fp.read(), auth=auth) + r.raise_for_status() + for name in dirs: + r = requests.put(target + rel_path + "/" + name, auth=auth) r.raise_for_status() - for name in dirs: - r = requests.put(target + rel_path + "/" + name, auth=auth) + + def get_circuitpython_version(self, device_url): + """ + Returns the version number of CircuitPython running on the board connected + via ``device_path``, along with the board ID. + + :param str device_url: http based device URL. + :return: A tuple with the version string for CircuitPython and the board ID string. + """ + url = urlparse(device_url) + return self._get_circuitpython_version_http(device_url) + + def _get_circuitpython_version(self, url): + """ + Returns the version number of CircuitPython running on the board connected + via ``device_path``, along with the board ID. This is obtained using + RESTful API from the /cp/version.json URL. + + :param str url: board URL. + :return: A tuple with the version string for CircuitPython and the board ID string. + """ + r = requests.get(url + "/cp/version.json") + # pylint: disable=no-member + if r.status_code != requests.codes.ok: + click.secho(f" Unable to get version from {url}: {r.status_code}", fg="red") + sys.exit(1) + # pylint: enable=no-member + ver_json = r.json() + return ver_json.get("version"), ver_json.get("board_id") + + def get_device_versions(self, device_url): + """ + Returns a dictionary of metadata from modules on the connected device. + + :param str device_url: URL for the device. + :return: A dictionary of metadata about the modules available on the + connected device. + """ + url = urlparse(device_url) + return self.get_modules(device_url + "/fs/lib/") + + def get_modules(self, device_url): + """ + Get a dictionary containing metadata about all the Python modules found in + the referenced path. + + :param str device_url: URL to be used to find modules. + :return: A dictionary containing metadata about the found modules. + """ + url = urlparse(device_url) + return self._get_modules_http(device_url) + + def _get_modules_http(self, url): + """ + Get a dictionary containing metadata about all the Python modules found using + the referenced URL. + + :param str url: URL for the modules. + :return: A dictionary containing metadata about the found modules. + """ + result = {} + u = urlparse(url) + auth = HTTPBasicAuth("", u.password) + r = requests.get(url, auth=auth, headers={"Accept": "application/json"}) + r.raise_for_status() + + directory_mods = [] + single_file_mods = [] + for entry in r.json(): + entry_name = entry.get("name") + if entry.get("directory"): + directory_mods.append(entry_name) + else: + if entry_name.endswith(".py") or entry_name.endswith(".mpy"): + single_file_mods.append(entry_name) + + self._get_modules_http_single_mods(auth, result, single_file_mods, url) + self._get_modules_http_dir_mods(auth, directory_mods, result, url) + + return result + + def _get_modules_http_dir_mods(self, auth, directory_mods, result, url): + """ + #TODO describe what this does + + :param auth HTTP authentication. + :param directory_mods list of modules. + :param result dictionary for the result. + :param url: URL of the device. + """ + for dm in directory_mods: + dm_url = url + dm + "/" + r = requests.get(dm_url, auth=auth, headers={"Accept": "application/json"}) + r.raise_for_status() + mpy = False + for entry in r.json(): + entry_name = entry.get("name") + if not entry.get("directory") and ( + entry_name.endswith(".py") or entry_name.endswith(".mpy") + ): + if entry_name.endswith(".mpy"): + mpy = True + r = requests.get(dm_url + entry_name, auth=auth) + r.raise_for_status() + idx = entry_name.rfind(".") + with tempfile.NamedTemporaryFile( + prefix=entry_name[:idx] + "-", suffix=entry_name[idx:], delete=False + ) as fp: + fp.write(r.content) + tmp_name = fp.name + metadata = extract_metadata(tmp_name) + os.remove(tmp_name) + if "__version__" in metadata: + metadata["path"] = dm_url + result[dm] = metadata + # break now if any of the submodules has a bad format + if metadata["__version__"] == BAD_FILE_FORMAT: + break + + if result.get(dm) is None: + result[dm] = {"path": dm_url, "mpy": mpy} + + def _get_modules_http_single_mods(self, auth, result, single_file_mods, url): + """ + :param auth HTTP authentication. + :param single_file_mods list of modules. + :param result dictionary for the result. + :param url: URL of the device. + """ + for sfm in single_file_mods: + sfm_url = url + sfm + r = requests.get(sfm_url, auth=auth) + r.raise_for_status() + idx = sfm.rfind(".") + with tempfile.NamedTemporaryFile( + prefix=sfm[:idx] + "-", suffix=sfm[idx:], delete=False + ) as fp: + fp.write(r.content) + tmp_name = fp.name + metadata = extract_metadata(tmp_name) + os.remove(tmp_name) + metadata["path"] = sfm_url + result[sfm[:idx]] = metadata + + # pylint: disable=too-many-locals,too-many-branches + def install_module( + self, device_path, device_modules, name, pyext, mod_names + ): # pragma: no cover + """ + Finds a connected device and installs a given module name if it + is available in the current module bundle and is not already + installed on the device. + TODO: There is currently no check for the version. + + :param str device_path: The path to the connected board. + :param list(dict) device_modules: List of module metadata from the device. + :param str name: Name of module to install + :param bool pyext: Boolean to specify if the module should be installed from + source or from a pre-compiled module + :param mod_names: Dictionary of metadata from modules that can be generated + with get_bundle_versions() + """ + if not name: + click.echo("No module name(s) provided.") + elif name in mod_names: + # Grab device modules to check if module already installed + if name in device_modules: + click.echo("'{}' is already installed.".format(name)) + return + + # Create the library directory first. + url = urlparse(device_path) + + library_path = device_path + "/fs/lib/" + auth = HTTPBasicAuth("", url.password) + r = requests.put(library_path, auth=auth) r.raise_for_status() + metadata = mod_names[name] + bundle = metadata["bundle"] + if pyext: + # Use Python source for module. + _install_module_py(library_path, metadata) + else: + # Use pre-compiled mpy modules. + self._install_module_mpy(bundle, library_path, metadata) + click.echo("Installed '{}'.".format(name)) + else: + click.echo("Unknown module named, '{}'.".format(name)) + + def _install_module_mpy(self, bundle, library_path, metadata): + """ + :param bundle library bundle. + :param library_path library path + :param metadata dictionary. + """ + url = urlparse(library_path) + module_name = os.path.basename(metadata["path"]).replace(".py", ".mpy") + if not module_name: + # Must be a directory based module. + module_name = os.path.basename(os.path.dirname(metadata["path"])) + major_version = CPY_VERSION.split(".")[0] + bundle_platform = "{}mpy".format(major_version) + bundle_path = os.path.join(bundle.lib_dir(bundle_platform), module_name) + if os.path.isdir(bundle_path): + + self.install_dir_http(bundle_path, library_path + module_name) + + elif os.path.isfile(bundle_path): + target = os.path.basename(bundle_path) + + self.install_file_http(bundle_path, library_path + target) + + else: + raise IOError("Cannot find compiled version of module.") + + # pylint: enable=too-many-locals,too-many-branches + def _install_module_py(self, library_path, metadata): + """ + :param library_path library path + :param metadata dictionary. + """ + url = urlparse(library_path) + source_path = metadata["path"] # Path to Python source version. + if os.path.isdir(source_path): + target = os.path.basename(os.path.dirname(source_path)) + + self.install_dir_http(source_path, library_path + target) + + else: + target = os.path.basename(source_path) + self.install_file_http(source_path, library_path + target) + + def libraries_from_imports(self, ctx, code_py, mod_names): + """ + Parse the given code.py file and return the imported libraries + + :param str code_py: Full path of the code.py file + :return: sequence of library names + """ + # pylint: disable=broad-except + + + url = code_py + auth = HTTPBasicAuth("", ctx.parent.params["password"]) + r = requests.get(url, auth=auth) + r.raise_for_status() + with open(LOCAL_CODE_PY_COPY, "w", encoding="utf-8") as f: + f.write(r.text) + + code_py = LOCAL_CODE_PY_COPY + + try: + found_imports = findimports.find_imports(code_py) + except Exception as ex: # broad exception because anything could go wrong + logger.exception(ex) + click.secho('Unable to read the auto file: "{}"'.format(str(ex)), fg="red") + sys.exit(2) + # pylint: enable=broad-except + imports = [info.name.split(".", 1)[0] for info in found_imports] + return [r for r in imports if r in mod_names] + + def _uninstall(self, device_path, module_path): + """ + Uninstall given module on device using REST API. + """ + url = urlparse(device_path) + auth = HTTPBasicAuth("", url.password) + r = requests.delete(module_path, auth=auth) + r.raise_for_status() + +class USBBackend: + def __init__(self): + pass + + + def get_circuitpython_version(self, device_url): + """ + Returns the version number of CircuitPython running on the board connected + via ``device_path``, along with the board ID. + + :param str device_url: device URL. filepath to the device + :return: A tuple with the version string for CircuitPython and the board ID string. + """ + url = urlparse(device_url) + return self._get_circuitpython_version_file(url.path) + + def _get_circuitpython_version_file(self, device_path): + """ + Returns the version number of CircuitPython running on the board connected + via ``device_path``, along with the board ID. This is obtained from the + ``boot_out.txt`` file on the device, whose first line will start with + something like this:: + + Adafruit CircuitPython 4.1.0 on 2019-08-02; + + While the second line is:: + + Board ID:raspberry_pi_pico + + :param str device_path: The path to the connected board. + :return: A tuple with the version string for CircuitPython and the board ID string. + """ + + try: + with open( + os.path.join(device_path, "boot_out.txt"), "r", encoding="utf-8" + ) as boot: + version_line = boot.readline() + circuit_python = version_line.split(";")[0].split(" ")[-3] + board_line = boot.readline() + if board_line.startswith("Board ID:"): + board_id = board_line[9:].strip() + else: + board_id = "" + except FileNotFoundError: + click.secho( + "Missing file boot_out.txt on the device: wrong path or drive corrupted.", + fg="red", + ) + logger.error("boot_out.txt not found.") + sys.exit(1) + + return circuit_python, board_id + + def get_device_versions(self, device_url): + """ + Returns a dictionary of metadata from modules on the connected device. + + :param str device_url: URL for the device. + :return: A dictionary of metadata about the modules available on the + connected device. + """ + url = urlparse(device_url) + return self.get_modules(os.path.join(url.path, "lib")) + + def get_modules(self, device_url): + """ + Get a dictionary containing metadata about all the Python modules found in + the referenced path. + + :param str device_url: URL to be used to find modules. + :return: A dictionary containing metadata about the found modules. + """ + url = urlparse(device_url) + return self._get_modules_file(device_url) + + def _get_modules_file(self, path): + """ + Get a dictionary containing metadata about all the Python modules found in + the referenced file system path. + + :param str path: The directory in which to find modules. + :return: A dictionary containing metadata about the found modules. + """ + result = {} + if not path: + return result + single_file_py_mods = glob.glob(os.path.join(path, "*.py")) + single_file_mpy_mods = glob.glob(os.path.join(path, "*.mpy")) + package_dir_mods = [ + d + for d in glob.glob(os.path.join(path, "*", "")) + if not os.path.basename(os.path.normpath(d)).startswith(".") + ] + single_file_mods = single_file_py_mods + single_file_mpy_mods + for sfm in [f for f in single_file_mods if not os.path.basename(f).startswith(".")]: + metadata = extract_metadata(sfm) + metadata["path"] = sfm + result[os.path.basename(sfm).replace(".py", "").replace(".mpy", "")] = metadata + for package_path in package_dir_mods: + name = os.path.basename(os.path.dirname(package_path)) + py_files = glob.glob(os.path.join(package_path, "**/*.py"), recursive=True) + mpy_files = glob.glob(os.path.join(package_path, "**/*.mpy"), recursive=True) + all_files = py_files + mpy_files + # default value + result[name] = {"path": package_path, "mpy": bool(mpy_files)} + # explore all the submodules to detect bad ones + for source in [f for f in all_files if not os.path.basename(f).startswith(".")]: + metadata = extract_metadata(source) + if "__version__" in metadata: + metadata["path"] = package_path + result[name] = metadata + # break now if any of the submodules has a bad format + if metadata["__version__"] == BAD_FILE_FORMAT: + break + return result + + # pylint: disable=too-many-locals,too-many-branches + def install_module( + self, device_path, device_modules, name, pyext, mod_names + ): # pragma: no cover + """ + Finds a connected device and installs a given module name if it + is available in the current module bundle and is not already + installed on the device. + TODO: There is currently no check for the version. + + :param str device_path: The path to the connected board. + :param list(dict) device_modules: List of module metadata from the device. + :param str name: Name of module to install + :param bool pyext: Boolean to specify if the module should be installed from + source or from a pre-compiled module + :param mod_names: Dictionary of metadata from modules that can be generated + with get_bundle_versions() + """ + if not name: + click.echo("No module name(s) provided.") + elif name in mod_names: + # Grab device modules to check if module already installed + if name in device_modules: + click.echo("'{}' is already installed.".format(name)) + return + + # Create the library directory first. + url = urlparse(device_path) + + library_path = os.path.join(device_path, "lib") + if not os.path.exists(library_path): # pragma: no cover + os.makedirs(library_path) + + metadata = mod_names[name] + bundle = metadata["bundle"] + if pyext: + # Use Python source for module. + _install_module_py(library_path, metadata) + else: + # Use pre-compiled mpy modules. + _install_module_mpy(bundle, library_path, metadata) + click.echo("Installed '{}'.".format(name)) + else: + click.echo("Unknown module named, '{}'.".format(name)) + + def _install_module_mpy(self, bundle, library_path, metadata): + """ + :param bundle library bundle. + :param library_path library path + :param metadata dictionary. + """ + url = urlparse(library_path) + module_name = os.path.basename(metadata["path"]).replace(".py", ".mpy") + if not module_name: + # Must be a directory based module. + module_name = os.path.basename(os.path.dirname(metadata["path"])) + major_version = CPY_VERSION.split(".")[0] + bundle_platform = "{}mpy".format(major_version) + bundle_path = os.path.join(bundle.lib_dir(bundle_platform), module_name) + if os.path.isdir(bundle_path): + target_path = os.path.join(library_path, module_name) + # Copy the directory. + shutil.copytree(bundle_path, target_path) + elif os.path.isfile(bundle_path): + target = os.path.basename(bundle_path) + target_path = os.path.join(library_path, target) + # Copy file. + shutil.copyfile(bundle_path, target_path) + else: + raise IOError("Cannot find compiled version of module.") + + # pylint: enable=too-many-locals,too-many-branches + def _install_module_py(self, library_path, metadata): + """ + :param library_path library path + :param metadata dictionary. + """ + url = urlparse(library_path) + source_path = metadata["path"] # Path to Python source version. + if os.path.isdir(source_path): + target = os.path.basename(os.path.dirname(source_path)) + target_path = os.path.join(library_path, target) + # Copy the directory. + shutil.copytree(source_path, target_path) + else: + target = os.path.basename(source_path) + target_path = os.path.join(library_path, target) + # Copy file. + shutil.copyfile(source_path, target_path) + + def libraries_from_imports(ctx, code_py, mod_names): + """ + Parse the given code.py file and return the imported libraries + + :param str code_py: Full path of the code.py file + :return: sequence of library names + """ + # pylint: disable=broad-except + # using_webworkflow = ( + # ctx.parent is not None + # and "host" in ctx.parent.params.keys() + # and ctx.parent.params["host"] is not None + # ) + + try: + found_imports = findimports.find_imports(code_py) + except Exception as ex: # broad exception because anything could go wrong + logger.exception(ex) + click.secho('Unable to read the auto file: "{}"'.format(str(ex)), fg="red") + sys.exit(2) + # pylint: enable=broad-except + imports = [info.name.split(".", 1)[0] for info in found_imports] + return [r for r in imports if r in mod_names] + + def _uninstall(device_path, module_path): + """ + Uninstall module using local file system. + """ + library_path = os.path.join(device_path, "lib") + if os.path.isdir(module_path): + target = os.path.basename(os.path.dirname(module_path)) + target_path = os.path.join(library_path, target) + # Remove the directory. + shutil.rmtree(target_path) + else: + target = os.path.basename(module_path) + target_path = os.path.join(library_path, target) + # Remove file + os.remove(target_path) + def clean_library_name(assumed_library_name): """ Most CP repos and library names are look like this: @@ -583,7 +1103,7 @@ def ensure_latest_bundle(bundle): def extract_metadata(path): """ - Given an file path, return a dictionary containing metadata extracted from + Given a file path, return a dictionary containing metadata extracted from dunder attributes found therein. Works with both .py and .mpy files. For Python source files, such metadata assignments should be simple and @@ -734,7 +1254,7 @@ def find_modules(device_url, bundles_list): """ # pylint: disable=broad-except,too-many-locals try: - device_modules = get_device_versions(device_url) + device_modules = backend.get_device_versions(device_url) bundle_modules = get_bundle_versions(bundles_list) result = [] for name, device_metadata in device_modules.items(): @@ -879,82 +1399,6 @@ def get_bundles_list(): return bundles_list -def get_circuitpython_version(device_url): - """ - Returns the version number of CircuitPython running on the board connected - via ``device_path``, along with the board ID. - - :param str device_url: device URL. Can be either file or http based. - :return: A tuple with the version string for CircuitPython and the board ID string. - """ - url = urlparse(device_url) - if url.scheme == "http": - return _get_circuitpython_version_http(device_url) - if url.scheme in ("", "file"): - return _get_circuitpython_version_file(url.path) - - click.secho(f"Not supported URL scheme: {url.scheme}", fg="red") - sys.exit(1) - - -def _get_circuitpython_version_http(url): - """ - Returns the version number of CircuitPython running on the board connected - via ``device_path``, along with the board ID. This is obtained using - RESTful API from the /cp/version.json URL. - - :param str url: board URL. - :return: A tuple with the version string for CircuitPython and the board ID string. - """ - r = requests.get(url + "/cp/version.json") - # pylint: disable=no-member - if r.status_code != requests.codes.ok: - click.secho(f" Unable to get version from {url}: {r.status_code}", fg="red") - sys.exit(1) - # pylint: enable=no-member - ver_json = r.json() - return ver_json.get("version"), ver_json.get("board_id") - - -def _get_circuitpython_version_file(device_path): - """ - Returns the version number of CircuitPython running on the board connected - via ``device_path``, along with the board ID. This is obtained from the - ``boot_out.txt`` file on the device, whose first line will start with - something like this:: - - Adafruit CircuitPython 4.1.0 on 2019-08-02; - - While the second line is:: - - Board ID:raspberry_pi_pico - - :param str device_path: The path to the connected board. - :return: A tuple with the version string for CircuitPython and the board ID string. - """ - - try: - with open( - os.path.join(device_path, "boot_out.txt"), "r", encoding="utf-8" - ) as boot: - version_line = boot.readline() - circuit_python = version_line.split(";")[0].split(" ")[-3] - board_line = boot.readline() - if board_line.startswith("Board ID:"): - board_id = board_line[9:].strip() - else: - board_id = "" - except FileNotFoundError: - click.secho( - "Missing file boot_out.txt on the device: wrong path or drive corrupted.", - fg="red", - ) - logger.error("boot_out.txt not found.") - sys.exit(1) - - return circuit_python, board_id - - def get_circup_version(): """Return the version of circup that is running. If not available, return None. @@ -975,7 +1419,8 @@ def get_circup_version(): def get_dependencies(*requested_libraries, mod_names, to_install=()): """ - Return a list of other CircuitPython libraries + Return a list of other CircuitPython libraries required by the given list + of libraries :param tuple requested_libraries: The libraries to search for dependencies :param object mod_names: All the modules metadata from bundle @@ -1068,36 +1513,6 @@ def get_circup_dependencies(bundle, library): return tuple() -def get_device_versions(device_url): - """ - Returns a dictionary of metadata from modules on the connected device. - - :param str device_url: URL for the device. - :return: A dictionary of metadata about the modules available on the - connected device. - """ - url = urlparse(device_url) - if url.scheme == "http": - return get_modules(device_url + "/fs/lib/") - - return get_modules(os.path.join(url.path, "lib")) - - -def get_modules(device_url): - """ - Get a dictionary containing metadata about all the Python modules found in - the referenced path. - - :param str device_url: URL to be used to find modules. - :return: A dictionary containing metadata about the found modules. - """ - url = urlparse(device_url) - if url.scheme == "http": - return _get_modules_http(device_url) - - return _get_modules_file(device_url) - - def get_latest_release_from_url(url): """ Find the tag name of the latest release by using HTTP HEAD and decoding the redirect. @@ -1116,287 +1531,6 @@ def get_latest_release_from_url(url): return tag -def _get_modules_http(url): - """ - Get a dictionary containing metadata about all the Python modules found using - the referenced URL. - - :param str url: URL for the modules. - :return: A dictionary containing metadata about the found modules. - """ - result = {} - u = urlparse(url) - auth = HTTPBasicAuth("", u.password) - r = requests.get(url, auth=auth, headers={"Accept": "application/json"}) - r.raise_for_status() - - directory_mods = [] - single_file_mods = [] - for entry in r.json(): - entry_name = entry.get("name") - if entry.get("directory"): - directory_mods.append(entry_name) - else: - if entry_name.endswith(".py") or entry_name.endswith(".mpy"): - single_file_mods.append(entry_name) - - _get_modules_http_single_mods(auth, result, single_file_mods, url) - _get_modules_http_dir_mods(auth, directory_mods, result, url) - - return result - - -def _get_modules_http_dir_mods(auth, directory_mods, result, url): - """ - :param auth HTTP authentication. - :param directory_mods list of modules. - :param result dictionary for the result. - :param url: URL of the device. - """ - for dm in directory_mods: - dm_url = url + dm + "/" - r = requests.get(dm_url, auth=auth, headers={"Accept": "application/json"}) - r.raise_for_status() - mpy = False - for entry in r.json(): - entry_name = entry.get("name") - if not entry.get("directory") and ( - entry_name.endswith(".py") or entry_name.endswith(".mpy") - ): - if entry_name.endswith(".mpy"): - mpy = True - r = requests.get(dm_url + entry_name, auth=auth) - r.raise_for_status() - idx = entry_name.rfind(".") - with tempfile.NamedTemporaryFile( - prefix=entry_name[:idx] + "-", suffix=entry_name[idx:], delete=False - ) as fp: - fp.write(r.content) - tmp_name = fp.name - metadata = extract_metadata(tmp_name) - os.remove(tmp_name) - if "__version__" in metadata: - metadata["path"] = dm_url - result[dm] = metadata - # break now if any of the submodules has a bad format - if metadata["__version__"] == BAD_FILE_FORMAT: - break - - if result.get(dm) is None: - result[dm] = {"path": dm_url, "mpy": mpy} - - -def _get_modules_http_single_mods(auth, result, single_file_mods, url): - """ - :param auth HTTP authentication. - :param single_file_mods list of modules. - :param result dictionary for the result. - :param url: URL of the device. - """ - for sfm in single_file_mods: - sfm_url = url + sfm - r = requests.get(sfm_url, auth=auth) - r.raise_for_status() - idx = sfm.rfind(".") - with tempfile.NamedTemporaryFile( - prefix=sfm[:idx] + "-", suffix=sfm[idx:], delete=False - ) as fp: - fp.write(r.content) - tmp_name = fp.name - metadata = extract_metadata(tmp_name) - os.remove(tmp_name) - metadata["path"] = sfm_url - result[sfm[:idx]] = metadata - - -def _get_modules_file(path): - """ - Get a dictionary containing metadata about all the Python modules found in - the referenced file system path. - - :param str path: The directory in which to find modules. - :return: A dictionary containing metadata about the found modules. - """ - result = {} - if not path: - return result - single_file_py_mods = glob.glob(os.path.join(path, "*.py")) - single_file_mpy_mods = glob.glob(os.path.join(path, "*.mpy")) - package_dir_mods = [ - d - for d in glob.glob(os.path.join(path, "*", "")) - if not os.path.basename(os.path.normpath(d)).startswith(".") - ] - single_file_mods = single_file_py_mods + single_file_mpy_mods - for sfm in [f for f in single_file_mods if not os.path.basename(f).startswith(".")]: - metadata = extract_metadata(sfm) - metadata["path"] = sfm - result[os.path.basename(sfm).replace(".py", "").replace(".mpy", "")] = metadata - for package_path in package_dir_mods: - name = os.path.basename(os.path.dirname(package_path)) - py_files = glob.glob(os.path.join(package_path, "**/*.py"), recursive=True) - mpy_files = glob.glob(os.path.join(package_path, "**/*.mpy"), recursive=True) - all_files = py_files + mpy_files - # default value - result[name] = {"path": package_path, "mpy": bool(mpy_files)} - # explore all the submodules to detect bad ones - for source in [f for f in all_files if not os.path.basename(f).startswith(".")]: - metadata = extract_metadata(source) - if "__version__" in metadata: - metadata["path"] = package_path - result[name] = metadata - # break now if any of the submodules has a bad format - if metadata["__version__"] == BAD_FILE_FORMAT: - break - return result - - -# pylint: disable=too-many-locals,too-many-branches -def install_module( - device_path, device_modules, name, pyext, mod_names -): # pragma: no cover - """ - Finds a connected device and installs a given module name if it - is available in the current module bundle and is not already - installed on the device. - TODO: There is currently no check for the version. - - :param str device_path: The path to the connected board. - :param list(dict) device_modules: List of module metadata from the device. - :param str name: Name of module to install - :param bool pyext: Boolean to specify if the module should be installed from - source or from a pre-compiled module - :param mod_names: Dictionary of metadata from modules that can be generated - with get_bundle_versions() - """ - if not name: - click.echo("No module name(s) provided.") - elif name in mod_names: - # Grab device modules to check if module already installed - if name in device_modules: - click.echo("'{}' is already installed.".format(name)) - return - - # Create the library directory first. - url = urlparse(device_path) - if url.scheme == "http": - library_path = device_path + "/fs/lib/" - auth = HTTPBasicAuth("", url.password) - r = requests.put(library_path, auth=auth) - r.raise_for_status() - else: - library_path = os.path.join(device_path, "lib") - if not os.path.exists(library_path): # pragma: no cover - os.makedirs(library_path) - - metadata = mod_names[name] - bundle = metadata["bundle"] - if pyext: - # Use Python source for module. - _install_module_py(library_path, metadata) - else: - # Use pre-compiled mpy modules. - _install_module_mpy(bundle, library_path, metadata) - click.echo("Installed '{}'.".format(name)) - else: - click.echo("Unknown module named, '{}'.".format(name)) - - -def _install_module_mpy(bundle, library_path, metadata): - """ - :param bundle library bundle. - :param library_path library path - :param metadata dictionary. - """ - url = urlparse(library_path) - module_name = os.path.basename(metadata["path"]).replace(".py", ".mpy") - if not module_name: - # Must be a directory based module. - module_name = os.path.basename(os.path.dirname(metadata["path"])) - major_version = CPY_VERSION.split(".")[0] - bundle_platform = "{}mpy".format(major_version) - bundle_path = os.path.join(bundle.lib_dir(bundle_platform), module_name) - if os.path.isdir(bundle_path): - if url.scheme == "http": - install_dir_http(bundle_path, library_path + module_name) - else: - target_path = os.path.join(library_path, module_name) - # Copy the directory. - shutil.copytree(bundle_path, target_path) - elif os.path.isfile(bundle_path): - target = os.path.basename(bundle_path) - if url.scheme == "http": - install_file_http(bundle_path, library_path + target) - else: - target_path = os.path.join(library_path, target) - # Copy file. - shutil.copyfile(bundle_path, target_path) - else: - raise IOError("Cannot find compiled version of module.") - - -def _install_module_py(library_path, metadata): - """ - :param library_path library path - :param metadata dictionary. - """ - url = urlparse(library_path) - source_path = metadata["path"] # Path to Python source version. - if os.path.isdir(source_path): - target = os.path.basename(os.path.dirname(source_path)) - if url.scheme == "http": - install_dir_http(source_path, library_path + target) - else: - target_path = os.path.join(library_path, target) - # Copy the directory. - shutil.copytree(source_path, target_path) - else: - target = os.path.basename(source_path) - if url.scheme == "http": - install_file_http(source_path, library_path + target) - else: - target_path = os.path.join(library_path, target) - # Copy file. - shutil.copyfile(source_path, target_path) - - -# pylint: enable=too-many-locals,too-many-branches - - -def libraries_from_imports(ctx, code_py, mod_names): - """ - Parse the given code.py file and return the imported libraries - - :param str code_py: Full path of the code.py file - :return: sequence of library names - """ - # pylint: disable=broad-except - if ctx is not None: - using_webworkflow = ( - ctx.parent is not None - and "host" in ctx.parent.params.keys() - and ctx.parent.params["host"] is not None - ) - if using_webworkflow: - url = code_py - auth = HTTPBasicAuth("", ctx.parent.params["password"]) - r = requests.get(url, auth=auth) - r.raise_for_status() - with open(LOCAL_CODE_PY_COPY, "w", encoding="utf-8") as f: - f.write(r.text) - - code_py = LOCAL_CODE_PY_COPY - try: - found_imports = findimports.find_imports(code_py) - except Exception as ex: # broad exception because anything could go wrong - logger.exception(ex) - click.secho('Unable to read the auto file: "{}"'.format(str(ex)), fg="red") - sys.exit(2) - # pylint: enable=broad-except - imports = [info.name.split(".", 1)[0] for info in found_imports] - return [r for r in imports if r in mod_names] - - def libraries_from_requirements(requirements): """ Clean up supplied requirements.txt and turn into tuple of CP libraries @@ -1515,6 +1649,7 @@ def main(ctx, verbose, path, host, password, board_id, cpy_version): # pragma: A tool to manage and update libraries on a CircuitPython device. """ ctx.ensure_object(dict) + if verbose: # Configure additional logging to stdout. global VERBOSE @@ -1795,42 +1930,15 @@ def uninstall(ctx, module): # pragma: no cover metadata = mod_names[name] module_path = metadata["path"] url = urlparse(device_path) - if url.scheme == "http": - _uninstall_http(device_path, module_path) - else: - _uninstall_file(device_path, module_path) + backend._uninstall(device_path, module_path) + # if url.scheme == "http": + # _uninstall_http(device_path, module_path) + # else: + # _uninstall_file(device_path, module_path) click.echo("Uninstalled '{}'.".format(name)) else: click.echo("Module '{}' not found on device.".format(name)) - -def _uninstall_http(device_path, module_path): - """ - Uninstall given module on device using REST API. - """ - url = urlparse(device_path) - auth = HTTPBasicAuth("", url.password) - r = requests.delete(module_path, auth=auth) - r.raise_for_status() - - -def _uninstall_file(device_path, module_path): - """ - Uninstall module using local file system. - """ - library_path = os.path.join(device_path, "lib") - if os.path.isdir(module_path): - target = os.path.basename(os.path.dirname(module_path)) - target_path = os.path.join(library_path, target) - # Remove the directory. - shutil.rmtree(target_path) - else: - target = os.path.basename(module_path) - target_path = os.path.join(library_path, target) - # Remove file - os.remove(target_path) - - # pylint: disable=too-many-branches From c6d430c0fcefc69d45030e9fa087b966958bf00f Mon Sep 17 00:00:00 2001 From: foamyguy Date: Fri, 17 Nov 2023 18:16:13 -0600 Subject: [PATCH 09/63] more refactoring --- circup/__init__.py | 44 ++++++++++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/circup/__init__.py b/circup/__init__.py index e9470b0..b8f77d5 100644 --- a/circup/__init__.py +++ b/circup/__init__.py @@ -676,7 +676,7 @@ def install_module( bundle = metadata["bundle"] if pyext: # Use Python source for module. - _install_module_py(library_path, metadata) + self._install_module_py(library_path, metadata) else: # Use pre-compiled mpy modules. self._install_module_mpy(bundle, library_path, metadata) @@ -919,10 +919,10 @@ def install_module( bundle = metadata["bundle"] if pyext: # Use Python source for module. - _install_module_py(library_path, metadata) + self._install_module_py(library_path, metadata) else: # Use pre-compiled mpy modules. - _install_module_mpy(bundle, library_path, metadata) + self._install_module_mpy(bundle, library_path, metadata) click.echo("Installed '{}'.".format(name)) else: click.echo("Unknown module named, '{}'.".format(name)) @@ -1650,6 +1650,18 @@ def main(ctx, verbose, path, host, password, board_id, cpy_version): # pragma: """ ctx.ensure_object(dict) + using_webworkflow = ( + ctx.parent is not None + and "host" in ctx.parent.params.keys() + and ctx.parent.params["host"] is not None + ) + + ctx.obj["using_webworkflow"] = using_webworkflow + if using_webworkflow: + ctx.obj["backend"] = WebBackend() + else: + ctx.obj["backend"] = USBBackend() + if verbose: # Configure additional logging to stdout. global VERBOSE @@ -1682,7 +1694,7 @@ def main(ctx, verbose, path, host, password, board_id, cpy_version): # pragma: sys.exit(1) else: CPY_VERSION, board_id = ( - get_circuitpython_version(device_path) + ctx.obj["backend"].get_circuitpython_version(device_path) if board_id is None or cpy_version is None else (cpy_version, board_id) ) @@ -1841,11 +1853,11 @@ def install(ctx, modules, pyext, requirement, auto, auto_file): # pragma: no co can be installed at once by providing more than one module name, each separated by a space. """ - using_webworkflow = ( - ctx.parent is not None - and "host" in ctx.parent.params.keys() - and ctx.parent.params["host"] is not None - ) + # using_webworkflow = ( + # ctx.parent is not None + # and "host" in ctx.parent.params.keys() + # and ctx.parent.params["host"] is not None + # ) # TODO: Ensure there's enough space on the device available_modules = get_bundle_versions(get_bundles_list()) mod_names = {} @@ -1861,27 +1873,27 @@ def install(ctx, modules, pyext, requirement, auto, auto_file): # pragma: no co # pass a local file with "./" or "../" is_relative = auto_file.split(os.sep)[0] in [os.path.curdir, os.path.pardir] if not os.path.isabs(auto_file) and not is_relative: - if not using_webworkflow: + if not ctx.obj["using_webworkflow"]: auto_file = os.path.join(ctx.obj["DEVICE_PATH"], auto_file or "code.py") else: auto_file = os.path.join( ctx.obj["DEVICE_PATH"], "fs", auto_file or "code.py" ) - if not os.path.isfile(auto_file) and not using_webworkflow: + if not os.path.isfile(auto_file) and not ctx.obj["using_webworkflow"]: click.secho(f"Auto file not found: {auto_file}", fg="red") sys.exit(1) - requested_installs = libraries_from_imports(ctx, auto_file, mod_names) + requested_installs = ctx.obj["backend"].libraries_from_imports(ctx, auto_file, mod_names) else: requested_installs = modules requested_installs = sorted(set(requested_installs)) click.echo(f"Searching for dependencies for: {requested_installs}") to_install = get_dependencies(requested_installs, mod_names=mod_names) - device_modules = get_device_versions(ctx.obj["DEVICE_PATH"]) + device_modules = ctx.obj["backend"].get_device_versions(ctx.obj["DEVICE_PATH"]) if to_install is not None: to_install = sorted(to_install) click.echo(f"Ready to install: {to_install}\n") for library in to_install: - install_module( + ctx.obj["backend"].install_module( ctx.obj["DEVICE_PATH"], device_modules, library, pyext, mod_names ) @@ -1921,7 +1933,7 @@ def uninstall(ctx, module): # pragma: no cover """ device_path = ctx.obj["DEVICE_PATH"] for name in module: - device_modules = get_device_versions(device_path) + device_modules = ctx.obj["backend"].get_device_versions(device_path) name = name.lower() mod_names = {} for module_item, metadata in device_modules.items(): @@ -1930,7 +1942,7 @@ def uninstall(ctx, module): # pragma: no cover metadata = mod_names[name] module_path = metadata["path"] url = urlparse(device_path) - backend._uninstall(device_path, module_path) + ctx.obj["backend"]._uninstall(device_path, module_path) # if url.scheme == "http": # _uninstall_http(device_path, module_path) # else: From 86ccabe411adfad08087d048b31cd9d9899c07af Mon Sep 17 00:00:00 2001 From: foamyguy Date: Mon, 20 Nov 2023 10:28:29 -0600 Subject: [PATCH 10/63] more refactoring, some successful USB workflow functionality --- circup/__init__.py | 308 ++++++++++++++++++++++++--------------------- 1 file changed, 163 insertions(+), 145 deletions(-) diff --git a/circup/__init__.py b/circup/__init__.py index b8f77d5..90bcb52 100644 --- a/circup/__init__.py +++ b/circup/__init__.py @@ -388,48 +388,6 @@ def row(self): update_reason = "Minor Version" return (self.name, loc, rem, update_reason) - def update(self): - """ - Delete the module on the device, then copy the module from the bundle - back onto the device. - - The caller is expected to handle any exceptions raised. - """ - url = urlparse(self.path) - if url.scheme == "http": - self._update_http() - else: - self._update_file() - - def _update_http(self): - """ - Update the module using web workflow. - """ - if self.file: - # Copy the file (will overwrite). - install_file_http(self.bundle_path, self.path) - else: - # Delete the directory (recursive) first. - url = urlparse(self.path) - auth = HTTPBasicAuth("", url.password) - r = requests.delete(self.path, auth=auth) - r.raise_for_status() - - install_dir_http(self.bundle_path, self.path) - - def _update_file(self): - """ - Update the module using file system. - """ - if os.path.isdir(self.path): - # Delete and copy the directory. - shutil.rmtree(self.path, ignore_errors=True) - shutil.copytree(self.bundle_path, self.path) - else: - # Delete and copy file. - os.remove(self.path) - shutil.copyfile(self.bundle_path, self.path) - def __repr__(self): """ Helps with log files. @@ -453,8 +411,13 @@ def __repr__(self): class WebBackend: - def __init__(self): - pass + """ + Backend for interacting with a device via Web Workflow + """ + + def __init__(self, host, password): + self.host = host + self.password = password def install_file_http(self, source, target): """ @@ -489,7 +452,9 @@ def install_dir_http(self, source, target): rel_path = "" for name in files: with open(os.path.join(root, name), "rb") as fp: - r = requests.put(target + rel_path + "/" + name, fp.read(), auth=auth) + r = requests.put( + target + rel_path + "/" + name, fp.read(), auth=auth + ) r.raise_for_status() for name in dirs: r = requests.put(target + rel_path + "/" + name, auth=auth) @@ -498,13 +463,12 @@ def install_dir_http(self, source, target): def get_circuitpython_version(self, device_url): """ Returns the version number of CircuitPython running on the board connected - via ``device_path``, along with the board ID. + via ``device_url``, along with the board ID. :param str device_url: http based device URL. :return: A tuple with the version string for CircuitPython and the board ID string. """ - url = urlparse(device_url) - return self._get_circuitpython_version_http(device_url) + return self._get_circuitpython_version(device_url) def _get_circuitpython_version(self, url): """ @@ -518,7 +482,9 @@ def _get_circuitpython_version(self, url): r = requests.get(url + "/cp/version.json") # pylint: disable=no-member if r.status_code != requests.codes.ok: - click.secho(f" Unable to get version from {url}: {r.status_code}", fg="red") + click.secho( + f" Unable to get version from {url}: {r.status_code}", fg="red" + ) sys.exit(1) # pylint: enable=no-member ver_json = r.json() @@ -532,7 +498,6 @@ def get_device_versions(self, device_url): :return: A dictionary of metadata about the modules available on the connected device. """ - url = urlparse(device_url) return self.get_modules(device_url + "/fs/lib/") def get_modules(self, device_url): @@ -543,7 +508,6 @@ def get_modules(self, device_url): :param str device_url: URL to be used to find modules. :return: A dictionary containing metadata about the found modules. """ - url = urlparse(device_url) return self._get_modules_http(device_url) def _get_modules_http(self, url): @@ -592,7 +556,7 @@ def _get_modules_http_dir_mods(self, auth, directory_mods, result, url): for entry in r.json(): entry_name = entry.get("name") if not entry.get("directory") and ( - entry_name.endswith(".py") or entry_name.endswith(".mpy") + entry_name.endswith(".py") or entry_name.endswith(".mpy") ): if entry_name.endswith(".mpy"): mpy = True @@ -600,7 +564,9 @@ def _get_modules_http_dir_mods(self, auth, directory_mods, result, url): r.raise_for_status() idx = entry_name.rfind(".") with tempfile.NamedTemporaryFile( - prefix=entry_name[:idx] + "-", suffix=entry_name[idx:], delete=False + prefix=entry_name[:idx] + "-", + suffix=entry_name[idx:], + delete=False, ) as fp: fp.write(r.content) tmp_name = fp.name @@ -629,7 +595,7 @@ def _get_modules_http_single_mods(self, auth, result, single_file_mods, url): r.raise_for_status() idx = sfm.rfind(".") with tempfile.NamedTemporaryFile( - prefix=sfm[:idx] + "-", suffix=sfm[idx:], delete=False + prefix=sfm[:idx] + "-", suffix=sfm[idx:], delete=False ) as fp: fp.write(r.content) tmp_name = fp.name @@ -638,9 +604,9 @@ def _get_modules_http_single_mods(self, auth, result, single_file_mods, url): metadata["path"] = sfm_url result[sfm[:idx]] = metadata - # pylint: disable=too-many-locals,too-many-branches + # pylint: disable=too-many-locals,too-many-branches,too-many-arguments def install_module( - self, device_path, device_modules, name, pyext, mod_names + self, device_path, device_modules, name, pyext, mod_names ): # pragma: no cover """ Finds a connected device and installs a given module name if it @@ -690,7 +656,6 @@ def _install_module_mpy(self, bundle, library_path, metadata): :param library_path library path :param metadata dictionary. """ - url = urlparse(library_path) module_name = os.path.basename(metadata["path"]).replace(".py", ".mpy") if not module_name: # Must be a directory based module. @@ -716,18 +681,16 @@ def _install_module_py(self, library_path, metadata): :param library_path library path :param metadata dictionary. """ - url = urlparse(library_path) source_path = metadata["path"] # Path to Python source version. if os.path.isdir(source_path): target = os.path.basename(os.path.dirname(source_path)) - self.install_dir_http(source_path, library_path + target) else: target = os.path.basename(source_path) self.install_file_http(source_path, library_path + target) - def libraries_from_imports(self, ctx, code_py, mod_names): + def libraries_from_imports(self, code_py, mod_names): """ Parse the given code.py file and return the imported libraries @@ -736,16 +699,15 @@ def libraries_from_imports(self, ctx, code_py, mod_names): """ # pylint: disable=broad-except - url = code_py - auth = HTTPBasicAuth("", ctx.parent.params["password"]) + auth = HTTPBasicAuth("", self.password) r = requests.get(url, auth=auth) r.raise_for_status() with open(LOCAL_CODE_PY_COPY, "w", encoding="utf-8") as f: f.write(r.text) code_py = LOCAL_CODE_PY_COPY - + try: found_imports = findimports.find_imports(code_py) except Exception as ex: # broad exception because anything could go wrong @@ -756,7 +718,7 @@ def libraries_from_imports(self, ctx, code_py, mod_names): imports = [info.name.split(".", 1)[0] for info in found_imports] return [r for r in imports if r in mod_names] - def _uninstall(self, device_path, module_path): + def uninstall(self, device_path, module_path): """ Uninstall given module on device using REST API. """ @@ -765,11 +727,95 @@ def _uninstall(self, device_path, module_path): r = requests.delete(module_path, auth=auth) r.raise_for_status() + def update(self, module): + """ + Delete the module on the device, then copy the module from the bundle + back onto the device. + + The caller is expected to handle any exceptions raised. + """ + url = urlparse(module.path) + if url.scheme == "http": + self._update_http(module) + + def _update_http(self, module): + """ + Update the module using web workflow. + """ + if module.file: + # Copy the file (will overwrite). + self.install_file_http(module.bundle_path, module.path) + else: + # Delete the directory (recursive) first. + url = urlparse(module.path) + auth = HTTPBasicAuth("", url.password) + r = requests.delete(module.path, auth=auth) + r.raise_for_status() + + self.install_dir_http(module.bundle_path, module.path) + + +def get_modules(device_url): + """ + Get a dictionary containing metadata about all the Python modules found in + the referenced path. + + :param str device_url: URL to be used to find modules. + :return: A dictionary containing metadata about the found modules. + """ + return _get_modules_file(device_url) + + +def _get_modules_file(path): + """ + Get a dictionary containing metadata about all the Python modules found in + the referenced file system path. + + :param str path: The directory in which to find modules. + :return: A dictionary containing metadata about the found modules. + """ + result = {} + if not path: + return result + single_file_py_mods = glob.glob(os.path.join(path, "*.py")) + single_file_mpy_mods = glob.glob(os.path.join(path, "*.mpy")) + package_dir_mods = [ + d + for d in glob.glob(os.path.join(path, "*", "")) + if not os.path.basename(os.path.normpath(d)).startswith(".") + ] + single_file_mods = single_file_py_mods + single_file_mpy_mods + for sfm in [f for f in single_file_mods if not os.path.basename(f).startswith(".")]: + metadata = extract_metadata(sfm) + metadata["path"] = sfm + result[os.path.basename(sfm).replace(".py", "").replace(".mpy", "")] = metadata + for package_path in package_dir_mods: + name = os.path.basename(os.path.dirname(package_path)) + py_files = glob.glob(os.path.join(package_path, "**/*.py"), recursive=True) + mpy_files = glob.glob(os.path.join(package_path, "**/*.mpy"), recursive=True) + all_files = py_files + mpy_files + # default value + result[name] = {"path": package_path, "mpy": bool(mpy_files)} + # explore all the submodules to detect bad ones + for source in [f for f in all_files if not os.path.basename(f).startswith(".")]: + metadata = extract_metadata(source) + if "__version__" in metadata: + metadata["path"] = package_path + result[name] = metadata + # break now if any of the submodules has a bad format + if metadata["__version__"] == BAD_FILE_FORMAT: + break + return result + + class USBBackend: + """ + Backend for interacting with a device via USB Workflow + """ + def __init__(self): pass - - + def get_circuitpython_version(self, device_url): """ Returns the version number of CircuitPython running on the board connected @@ -800,7 +846,7 @@ def _get_circuitpython_version_file(self, device_path): try: with open( - os.path.join(device_path, "boot_out.txt"), "r", encoding="utf-8" + os.path.join(device_path, "boot_out.txt"), "r", encoding="utf-8" ) as boot: version_line = boot.readline() circuit_python = version_line.split(";")[0].split(" ")[-3] @@ -828,63 +874,11 @@ def get_device_versions(self, device_url): connected device. """ url = urlparse(device_url) - return self.get_modules(os.path.join(url.path, "lib")) - - def get_modules(self, device_url): - """ - Get a dictionary containing metadata about all the Python modules found in - the referenced path. - - :param str device_url: URL to be used to find modules. - :return: A dictionary containing metadata about the found modules. - """ - url = urlparse(device_url) - return self._get_modules_file(device_url) - - def _get_modules_file(self, path): - """ - Get a dictionary containing metadata about all the Python modules found in - the referenced file system path. + return get_modules(os.path.join(url.path, "lib")) - :param str path: The directory in which to find modules. - :return: A dictionary containing metadata about the found modules. - """ - result = {} - if not path: - return result - single_file_py_mods = glob.glob(os.path.join(path, "*.py")) - single_file_mpy_mods = glob.glob(os.path.join(path, "*.mpy")) - package_dir_mods = [ - d - for d in glob.glob(os.path.join(path, "*", "")) - if not os.path.basename(os.path.normpath(d)).startswith(".") - ] - single_file_mods = single_file_py_mods + single_file_mpy_mods - for sfm in [f for f in single_file_mods if not os.path.basename(f).startswith(".")]: - metadata = extract_metadata(sfm) - metadata["path"] = sfm - result[os.path.basename(sfm).replace(".py", "").replace(".mpy", "")] = metadata - for package_path in package_dir_mods: - name = os.path.basename(os.path.dirname(package_path)) - py_files = glob.glob(os.path.join(package_path, "**/*.py"), recursive=True) - mpy_files = glob.glob(os.path.join(package_path, "**/*.mpy"), recursive=True) - all_files = py_files + mpy_files - # default value - result[name] = {"path": package_path, "mpy": bool(mpy_files)} - # explore all the submodules to detect bad ones - for source in [f for f in all_files if not os.path.basename(f).startswith(".")]: - metadata = extract_metadata(source) - if "__version__" in metadata: - metadata["path"] = package_path - result[name] = metadata - # break now if any of the submodules has a bad format - if metadata["__version__"] == BAD_FILE_FORMAT: - break - return result - - # pylint: disable=too-many-locals,too-many-branches + # pylint: disable=too-many-locals,too-many-branches,too-many-arguments def install_module( - self, device_path, device_modules, name, pyext, mod_names + self, device_path, device_modules, name, pyext, mod_names ): # pragma: no cover """ Finds a connected device and installs a given module name if it @@ -909,8 +903,6 @@ def install_module( return # Create the library directory first. - url = urlparse(device_path) - library_path = os.path.join(device_path, "lib") if not os.path.exists(library_path): # pragma: no cover os.makedirs(library_path) @@ -933,7 +925,6 @@ def _install_module_mpy(self, bundle, library_path, metadata): :param library_path library path :param metadata dictionary. """ - url = urlparse(library_path) module_name = os.path.basename(metadata["path"]).replace(".py", ".mpy") if not module_name: # Must be a directory based module. @@ -959,7 +950,7 @@ def _install_module_py(self, library_path, metadata): :param library_path library path :param metadata dictionary. """ - url = urlparse(library_path) + source_path = metadata["path"] # Path to Python source version. if os.path.isdir(source_path): target = os.path.basename(os.path.dirname(source_path)) @@ -972,7 +963,7 @@ def _install_module_py(self, library_path, metadata): # Copy file. shutil.copyfile(source_path, target_path) - def libraries_from_imports(ctx, code_py, mod_names): + def libraries_from_imports(self, code_py, mod_names): """ Parse the given code.py file and return the imported libraries @@ -997,7 +988,7 @@ def libraries_from_imports(ctx, code_py, mod_names): imports = [info.name.split(".", 1)[0] for info in found_imports] return [r for r in imports if r in mod_names] - def _uninstall(device_path, module_path): + def uninstall(self, device_path, module_path): """ Uninstall module using local file system. """ @@ -1012,7 +1003,30 @@ def _uninstall(device_path, module_path): target_path = os.path.join(library_path, target) # Remove file os.remove(target_path) - + + def update(self, module): + """ + Delete the module on the device, then copy the module from the bundle + back onto the device. + + The caller is expected to handle any exceptions raised. + """ + self._update_file(module) + + def _update_file(self, module): + """ + Update the module using file system. + """ + if os.path.isdir(module.path): + # Delete and copy the directory. + shutil.rmtree(module.path, ignore_errors=True) + shutil.copytree(module.bundle_path, module.path) + else: + # Delete and copy file. + os.remove(module.path) + shutil.copyfile(module.bundle_path, module.path) + + def clean_library_name(assumed_library_name): """ Most CP repos and library names are look like this: @@ -1241,14 +1255,14 @@ def get_volume_name(disk_name): return device_dir -def find_modules(device_url, bundles_list): +def find_modules(backend, device_url, bundles_list): """ Extracts metadata from the connected device and available bundles and returns this as a list of Module instances representing the modules on the device. :param str device_url: The URL to the board. - :param Bundle bundles_list: List of supported bundles as Bundle objects. + :param List[Bundle] bundles_list: List of supported bundles as Bundle objects. :return: A list of Module instances describing the current state of the modules on the connected device. """ @@ -1649,16 +1663,16 @@ def main(ctx, verbose, path, host, password, board_id, cpy_version): # pragma: A tool to manage and update libraries on a CircuitPython device. """ ctx.ensure_object(dict) - + using_webworkflow = ( ctx.parent is not None and "host" in ctx.parent.params.keys() and ctx.parent.params["host"] is not None ) - + ctx.obj["using_webworkflow"] = using_webworkflow if using_webworkflow: - ctx.obj["backend"] = WebBackend() + ctx.obj["backend"] = WebBackend(host=host, password=password) else: ctx.obj["backend"] = USBBackend() @@ -1758,7 +1772,9 @@ def freeze(ctx, requirement): # pragma: no cover device. Option -r saves output to requirements.txt file """ logger.info("Freeze") - modules = find_modules(ctx.obj["DEVICE_PATH"], get_bundles_list()) + modules = find_modules( + ctx.obj["backend"], ctx.obj["DEVICE_PATH"], get_bundles_list() + ) if modules: output = [] for module in modules: @@ -1791,7 +1807,9 @@ def list_cli(ctx): # pragma: no cover modules = [ m.row - for m in find_modules(ctx.obj["DEVICE_PATH"], get_bundles_list()) + for m in find_modules( + ctx.obj["backend"], ctx.obj["DEVICE_PATH"], get_bundles_list() + ) if m.outofdate ] if modules: @@ -1882,7 +1900,9 @@ def install(ctx, modules, pyext, requirement, auto, auto_file): # pragma: no co if not os.path.isfile(auto_file) and not ctx.obj["using_webworkflow"]: click.secho(f"Auto file not found: {auto_file}", fg="red") sys.exit(1) - requested_installs = ctx.obj["backend"].libraries_from_imports(ctx, auto_file, mod_names) + requested_installs = ctx.obj["backend"].libraries_from_imports( + auto_file, mod_names + ) else: requested_installs = modules requested_installs = sorted(set(requested_installs)) @@ -1941,16 +1961,12 @@ def uninstall(ctx, module): # pragma: no cover if name in mod_names: metadata = mod_names[name] module_path = metadata["path"] - url = urlparse(device_path) - ctx.obj["backend"]._uninstall(device_path, module_path) - # if url.scheme == "http": - # _uninstall_http(device_path, module_path) - # else: - # _uninstall_file(device_path, module_path) + ctx.obj["backend"].uninstall(device_path, module_path) click.echo("Uninstalled '{}'.".format(name)) else: click.echo("Module '{}' not found on device.".format(name)) + # pylint: disable=too-many-branches @@ -1976,7 +1992,9 @@ def update(ctx, update_all): # pragma: no cover # Grab out of date modules. modules = [ m - for m in find_modules(ctx.obj["DEVICE_PATH"], get_bundles_list()) + for m in find_modules( + ctx.obj["backend"], ctx.obj["DEVICE_PATH"], get_bundles_list() + ) if m.outofdate ] if modules: @@ -2029,7 +2047,7 @@ def update(ctx, update_all): # pragma: no cover if update_flag: # pylint: disable=broad-except try: - module.update() + ctx.obj["backend"].update(module) click.echo("Updated {}".format(module.name)) except Exception as ex: logger.exception(ex) From 90299a099a95491e66a49d8a29c6b5591073f065 Mon Sep 17 00:00:00 2001 From: foamyguy Date: Mon, 20 Nov 2023 11:01:58 -0600 Subject: [PATCH 11/63] fix --path argument functionality --- circup/__init__.py | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/circup/__init__.py b/circup/__init__.py index 90bcb52..54d8b15 100644 --- a/circup/__init__.py +++ b/circup/__init__.py @@ -972,12 +972,6 @@ def libraries_from_imports(self, code_py, mod_names): """ # pylint: disable=broad-except - # using_webworkflow = ( - # ctx.parent is not None - # and "host" in ctx.parent.params.keys() - # and ctx.parent.params["host"] is not None - # ) - try: found_imports = findimports.find_imports(code_py) except Exception as ex: # broad exception because anything could go wrong @@ -1345,7 +1339,7 @@ def get_bundle_versions(bundles_list, avoid_download=False): of the library bundle. Uses the Python version (rather than the compiled version) of the library modules. - :param Bundle bundles_list: List of supported bundles as Bundle objects. + :param List[Bundle] bundles_list: List of supported bundles as Bundle objects. :param bool avoid_download: if True, download the bundle only if missing. :return: A dictionary of metadata about the modules available in the library bundle. @@ -1743,7 +1737,7 @@ def get_device_path(host, password, path): :return device URL or None if the device cannot be found. """ if path: - device_path = "file:///" + path + device_path = path elif host: if password is None: click.secho("--host needs --password", fg="red") @@ -1871,11 +1865,7 @@ def install(ctx, modules, pyext, requirement, auto, auto_file): # pragma: no co can be installed at once by providing more than one module name, each separated by a space. """ - # using_webworkflow = ( - # ctx.parent is not None - # and "host" in ctx.parent.params.keys() - # and ctx.parent.params["host"] is not None - # ) + # TODO: Ensure there's enough space on the device available_modules = get_bundle_versions(get_bundles_list()) mod_names = {} From c1af505510f44dec45b4e3b69ed35adadedf89e5 Mon Sep 17 00:00:00 2001 From: foamyguy Date: Tue, 21 Nov 2023 17:36:56 -0600 Subject: [PATCH 12/63] main command function arguments access --- circup/__init__.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/circup/__init__.py b/circup/__init__.py index 54d8b15..6d50e46 100644 --- a/circup/__init__.py +++ b/circup/__init__.py @@ -442,7 +442,9 @@ def install_dir_http(self, source, target): auth = HTTPBasicAuth("", url.password) # Create the top level directory. + print(f'target: {target + "/"}') r = requests.put(target + "/", auth=auth) + print(f"status: {r.status_code}") r.raise_for_status() # Traverse the directory structure and create the directories/files. @@ -742,6 +744,7 @@ def _update_http(self, module): """ Update the module using web workflow. """ + print(module.bundle_path) if module.file: # Copy the file (will overwrite). self.install_file_http(module.bundle_path, module.path) @@ -749,9 +752,11 @@ def _update_http(self, module): # Delete the directory (recursive) first. url = urlparse(module.path) auth = HTTPBasicAuth("", url.password) + r = requests.delete(module.path, auth=auth) + r.raise_for_status() - + print(module.path) self.install_dir_http(module.bundle_path, module.path) @@ -1659,9 +1664,8 @@ def main(ctx, verbose, path, host, password, board_id, cpy_version): # pragma: ctx.ensure_object(dict) using_webworkflow = ( - ctx.parent is not None - and "host" in ctx.parent.params.keys() - and ctx.parent.params["host"] is not None + "host" in ctx.params.keys() + and ctx.params["host"] is not None ) ctx.obj["using_webworkflow"] = using_webworkflow From 1b2e115155e27f3f8696ed43c3e04207e886207a Mon Sep 17 00:00:00 2001 From: foamyguy Date: Tue, 21 Nov 2023 18:04:21 -0600 Subject: [PATCH 13/63] remove prints, fix trailing slash difference --- circup/__init__.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/circup/__init__.py b/circup/__init__.py index 6d50e46..d38b49c 100644 --- a/circup/__init__.py +++ b/circup/__init__.py @@ -442,9 +442,7 @@ def install_dir_http(self, source, target): auth = HTTPBasicAuth("", url.password) # Create the top level directory. - print(f'target: {target + "/"}') - r = requests.put(target + "/", auth=auth) - print(f"status: {r.status_code}") + r = requests.put(target + ("/" if not target.endswith("/") else ""), auth=auth) r.raise_for_status() # Traverse the directory structure and create the directories/files. @@ -744,7 +742,6 @@ def _update_http(self, module): """ Update the module using web workflow. """ - print(module.bundle_path) if module.file: # Copy the file (will overwrite). self.install_file_http(module.bundle_path, module.path) @@ -752,11 +749,8 @@ def _update_http(self, module): # Delete the directory (recursive) first. url = urlparse(module.path) auth = HTTPBasicAuth("", url.password) - r = requests.delete(module.path, auth=auth) - r.raise_for_status() - print(module.path) self.install_dir_http(module.bundle_path, module.path) From 01a1ac2661ec9c8b6e36a64dbf1138998995653c Mon Sep 17 00:00:00 2001 From: foamyguy Date: Tue, 21 Nov 2023 18:10:01 -0600 Subject: [PATCH 14/63] code format --- circup/__init__.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/circup/__init__.py b/circup/__init__.py index d38b49c..6070205 100644 --- a/circup/__init__.py +++ b/circup/__init__.py @@ -1657,10 +1657,7 @@ def main(ctx, verbose, path, host, password, board_id, cpy_version): # pragma: """ ctx.ensure_object(dict) - using_webworkflow = ( - "host" in ctx.params.keys() - and ctx.params["host"] is not None - ) + using_webworkflow = "host" in ctx.params.keys() and ctx.params["host"] is not None ctx.obj["using_webworkflow"] = using_webworkflow if using_webworkflow: From f582873328976904a19933c2247f6feaf8473fa5 Mon Sep 17 00:00:00 2001 From: foamyguy Date: Tue, 21 Nov 2023 19:17:43 -0600 Subject: [PATCH 15/63] Backend super class. --- circup/__init__.py | 354 +++++++++++++++++++++++---------------------- 1 file changed, 178 insertions(+), 176 deletions(-) diff --git a/circup/__init__.py b/circup/__init__.py index 6070205..faa72a5 100644 --- a/circup/__init__.py +++ b/circup/__init__.py @@ -410,12 +410,164 @@ def __repr__(self): ) -class WebBackend: +class Backend: + """ + Backend parent class to be extended for workflow specific + implementations + """ + + def __init__(self): + self.LIB_DIR_PATH = None + + def _get_circuitpython_version(self, device_location): + """ + To be overridden by subclass + """ + raise NotImplementedError + + def get_circuitpython_version(self, device_location): + """ + Returns the version number of CircuitPython running on the board connected + via ``device_url``, along with the board ID. + + :param str device_location: http based device URL or local file path. + :return: A tuple with the version string for CircuitPython and the board ID string. + """ + return self._get_circuitpython_version(device_location) + + def _get_modules(self, device_lib_path): + """ + To be overridden by subclass + """ + raise NotImplementedError + + def get_modules(self, device_url): + """ + Get a dictionary containing metadata about all the Python modules found in + the referenced path. + + :param str device_url: URL to be used to find modules. + :return: A dictionary containing metadata about the found modules. + """ + return self._get_modules(device_url) + + def get_device_versions(self, device_url): + """ + Returns a dictionary of metadata from modules on the connected device. + + :param str device_url: URL for the device. + :return: A dictionary of metadata about the modules available on the + connected device. + """ + # url = urlparse(device_url) + return self.get_modules(os.path.join(device_url, self.LIB_DIR_PATH)) + + def _create_library_directory(self, device_path, library_path): + """ + To be overridden by subclass + """ + raise NotImplementedError + + def _install_module_py(self, library_path, metadata): + """ + To be overridden by subclass + """ + raise NotImplementedError + + def _install_module_mpy(self, bundle, library_path, metadata): + """ + To be overridden by subclass + """ + raise NotImplementedError + + # pylint: disable=too-many-locals,too-many-branches,too-many-arguments + def install_module( + self, device_path, device_modules, name, pyext, mod_names + ): # pragma: no cover + """ + Finds a connected device and installs a given module name if it + is available in the current module bundle and is not already + installed on the device. + TODO: There is currently no check for the version. + + :param str device_path: The path to the connected board. + :param list(dict) device_modules: List of module metadata from the device. + :param str name: Name of module to install + :param bool pyext: Boolean to specify if the module should be installed from + source or from a pre-compiled module + :param mod_names: Dictionary of metadata from modules that can be generated + with get_bundle_versions() + """ + if not name: + click.echo("No module name(s) provided.") + elif name in mod_names: + # Grab device modules to check if module already installed + if name in device_modules: + click.echo("'{}' is already installed.".format(name)) + return + + library_path = os.path.join(device_path, self.LIB_DIR_PATH) + + # Create the library directory first. + self._create_library_directory(device_path, library_path) + + metadata = mod_names[name] + bundle = metadata["bundle"] + if pyext: + # Use Python source for module. + self._install_module_py(library_path, metadata) + else: + # Use pre-compiled mpy modules. + self._install_module_mpy(bundle, library_path, metadata) + click.echo("Installed '{}'.".format(name)) + else: + click.echo("Unknown module named, '{}'.".format(name)) + + def libraries_from_imports(self, code_py, mod_names): + """ + To be overridden by subclass + """ + raise NotImplementedError + + def uninstall(self, device_path, module_path): + """ + To be overridden by subclass + """ + raise NotImplementedError + + def update(self, module): + """ + To be overridden by subclass + """ + raise NotImplementedError + + def libraries_from_code_py(self, code_py, mod_names): + """ + Parse the given code.py file and return the imported libraries + + :param str code_py: Full path of the code.py file + :return: sequence of library names + """ + # pylint: disable=broad-except + try: + found_imports = findimports.find_imports(code_py) + except Exception as ex: # broad exception because anything could go wrong + logger.exception(ex) + click.secho('Unable to read the auto file: "{}"'.format(str(ex)), fg="red") + sys.exit(2) + # pylint: enable=broad-except + imports = [info.name.split(".", 1)[0] for info in found_imports] + return [r for r in imports if r in mod_names] + + +class WebBackend(Backend): """ Backend for interacting with a device via Web Workflow """ def __init__(self, host, password): + super().__init__() + self.LIB_DIR_PATH = "fs/lib/" self.host = host self.password = password @@ -460,16 +612,6 @@ def install_dir_http(self, source, target): r = requests.put(target + rel_path + "/" + name, auth=auth) r.raise_for_status() - def get_circuitpython_version(self, device_url): - """ - Returns the version number of CircuitPython running on the board connected - via ``device_url``, along with the board ID. - - :param str device_url: http based device URL. - :return: A tuple with the version string for CircuitPython and the board ID string. - """ - return self._get_circuitpython_version(device_url) - def _get_circuitpython_version(self, url): """ Returns the version number of CircuitPython running on the board connected @@ -479,6 +621,7 @@ def _get_circuitpython_version(self, url): :param str url: board URL. :return: A tuple with the version string for CircuitPython and the board ID string. """ + # pylint: disable=arguments-renamed r = requests.get(url + "/cp/version.json") # pylint: disable=no-member if r.status_code != requests.codes.ok: @@ -490,25 +633,8 @@ def _get_circuitpython_version(self, url): ver_json = r.json() return ver_json.get("version"), ver_json.get("board_id") - def get_device_versions(self, device_url): - """ - Returns a dictionary of metadata from modules on the connected device. - - :param str device_url: URL for the device. - :return: A dictionary of metadata about the modules available on the - connected device. - """ - return self.get_modules(device_url + "/fs/lib/") - - def get_modules(self, device_url): - """ - Get a dictionary containing metadata about all the Python modules found in - the referenced path. - - :param str device_url: URL to be used to find modules. - :return: A dictionary containing metadata about the found modules. - """ - return self._get_modules_http(device_url) + def _get_modules(self, device_lib_path): + return self._get_modules_http(device_lib_path) def _get_modules_http(self, url): """ @@ -604,51 +730,11 @@ def _get_modules_http_single_mods(self, auth, result, single_file_mods, url): metadata["path"] = sfm_url result[sfm[:idx]] = metadata - # pylint: disable=too-many-locals,too-many-branches,too-many-arguments - def install_module( - self, device_path, device_modules, name, pyext, mod_names - ): # pragma: no cover - """ - Finds a connected device and installs a given module name if it - is available in the current module bundle and is not already - installed on the device. - TODO: There is currently no check for the version. - - :param str device_path: The path to the connected board. - :param list(dict) device_modules: List of module metadata from the device. - :param str name: Name of module to install - :param bool pyext: Boolean to specify if the module should be installed from - source or from a pre-compiled module - :param mod_names: Dictionary of metadata from modules that can be generated - with get_bundle_versions() - """ - if not name: - click.echo("No module name(s) provided.") - elif name in mod_names: - # Grab device modules to check if module already installed - if name in device_modules: - click.echo("'{}' is already installed.".format(name)) - return - - # Create the library directory first. - url = urlparse(device_path) - - library_path = device_path + "/fs/lib/" - auth = HTTPBasicAuth("", url.password) - r = requests.put(library_path, auth=auth) - r.raise_for_status() - - metadata = mod_names[name] - bundle = metadata["bundle"] - if pyext: - # Use Python source for module. - self._install_module_py(library_path, metadata) - else: - # Use pre-compiled mpy modules. - self._install_module_mpy(bundle, library_path, metadata) - click.echo("Installed '{}'.".format(name)) - else: - click.echo("Unknown module named, '{}'.".format(name)) + def _create_library_directory(self, device_path, library_path): + url = urlparse(device_path) + auth = HTTPBasicAuth("", url.password) + r = requests.put(library_path, auth=auth) + r.raise_for_status() def _install_module_mpy(self, bundle, library_path, metadata): """ @@ -697,26 +783,15 @@ def libraries_from_imports(self, code_py, mod_names): :param str code_py: Full path of the code.py file :return: sequence of library names """ - # pylint: disable=broad-except - url = code_py auth = HTTPBasicAuth("", self.password) r = requests.get(url, auth=auth) r.raise_for_status() with open(LOCAL_CODE_PY_COPY, "w", encoding="utf-8") as f: f.write(r.text) - code_py = LOCAL_CODE_PY_COPY - try: - found_imports = findimports.find_imports(code_py) - except Exception as ex: # broad exception because anything could go wrong - logger.exception(ex) - click.secho('Unable to read the auto file: "{}"'.format(str(ex)), fg="red") - sys.exit(2) - # pylint: enable=broad-except - imports = [info.name.split(".", 1)[0] for info in found_imports] - return [r for r in imports if r in mod_names] + return self.libraries_from_code_py(code_py, mod_names) def uninstall(self, device_path, module_path): """ @@ -734,9 +809,7 @@ def update(self, module): The caller is expected to handle any exceptions raised. """ - url = urlparse(module.path) - if url.scheme == "http": - self._update_http(module) + self._update_http(module) def _update_http(self, module): """ @@ -754,17 +827,6 @@ def _update_http(self, module): self.install_dir_http(module.bundle_path, module.path) -def get_modules(device_url): - """ - Get a dictionary containing metadata about all the Python modules found in - the referenced path. - - :param str device_url: URL to be used to find modules. - :return: A dictionary containing metadata about the found modules. - """ - return _get_modules_file(device_url) - - def _get_modules_file(path): """ Get a dictionary containing metadata about all the Python modules found in @@ -807,26 +869,16 @@ def _get_modules_file(path): return result -class USBBackend: +class USBBackend(Backend): """ Backend for interacting with a device via USB Workflow """ def __init__(self): - pass - - def get_circuitpython_version(self, device_url): - """ - Returns the version number of CircuitPython running on the board connected - via ``device_path``, along with the board ID. - - :param str device_url: device URL. filepath to the device - :return: A tuple with the version string for CircuitPython and the board ID string. - """ - url = urlparse(device_url) - return self._get_circuitpython_version_file(url.path) + self.LIB_DIR_PATH = "lib" + super().__init__() - def _get_circuitpython_version_file(self, device_path): + def _get_circuitpython_version(self, device_location): """ Returns the version number of CircuitPython running on the board connected via ``device_path``, along with the board ID. This is obtained from the @@ -842,7 +894,7 @@ def _get_circuitpython_version_file(self, device_path): :param str device_path: The path to the connected board. :return: A tuple with the version string for CircuitPython and the board ID string. """ - + device_path = urlparse(device_location).path try: with open( os.path.join(device_path, "boot_out.txt"), "r", encoding="utf-8" @@ -864,59 +916,19 @@ def _get_circuitpython_version_file(self, device_path): return circuit_python, board_id - def get_device_versions(self, device_url): - """ - Returns a dictionary of metadata from modules on the connected device. - - :param str device_url: URL for the device. - :return: A dictionary of metadata about the modules available on the - connected device. - """ - url = urlparse(device_url) - return get_modules(os.path.join(url.path, "lib")) - - # pylint: disable=too-many-locals,too-many-branches,too-many-arguments - def install_module( - self, device_path, device_modules, name, pyext, mod_names - ): # pragma: no cover + def _get_modules(self, device_lib_path): """ - Finds a connected device and installs a given module name if it - is available in the current module bundle and is not already - installed on the device. - TODO: There is currently no check for the version. + Get a dictionary containing metadata about all the Python modules found in + the referenced path. - :param str device_path: The path to the connected board. - :param list(dict) device_modules: List of module metadata from the device. - :param str name: Name of module to install - :param bool pyext: Boolean to specify if the module should be installed from - source or from a pre-compiled module - :param mod_names: Dictionary of metadata from modules that can be generated - with get_bundle_versions() + :param str device_lib_path: URL to be used to find modules. + :return: A dictionary containing metadata about the found modules. """ - if not name: - click.echo("No module name(s) provided.") - elif name in mod_names: - # Grab device modules to check if module already installed - if name in device_modules: - click.echo("'{}' is already installed.".format(name)) - return - - # Create the library directory first. - library_path = os.path.join(device_path, "lib") - if not os.path.exists(library_path): # pragma: no cover - os.makedirs(library_path) + return _get_modules_file(device_lib_path) - metadata = mod_names[name] - bundle = metadata["bundle"] - if pyext: - # Use Python source for module. - self._install_module_py(library_path, metadata) - else: - # Use pre-compiled mpy modules. - self._install_module_mpy(bundle, library_path, metadata) - click.echo("Installed '{}'.".format(name)) - else: - click.echo("Unknown module named, '{}'.".format(name)) + def _create_library_directory(self, device_path, library_path): + if not os.path.exists(library_path): # pragma: no cover + os.makedirs(library_path) def _install_module_mpy(self, bundle, library_path, metadata): """ @@ -969,17 +981,7 @@ def libraries_from_imports(self, code_py, mod_names): :param str code_py: Full path of the code.py file :return: sequence of library names """ - # pylint: disable=broad-except - - try: - found_imports = findimports.find_imports(code_py) - except Exception as ex: # broad exception because anything could go wrong - logger.exception(ex) - click.secho('Unable to read the auto file: "{}"'.format(str(ex)), fg="red") - sys.exit(2) - # pylint: enable=broad-except - imports = [info.name.split(".", 1)[0] for info in found_imports] - return [r for r in imports if r in mod_names] + return self.libraries_from_code_py(code_py, mod_names) def uninstall(self, device_path, module_path): """ From 1681b29f9e0b335ba411ceca127731ddf1f2bcd1 Mon Sep 17 00:00:00 2001 From: foamyguy Date: Wed, 22 Nov 2023 22:37:37 -0600 Subject: [PATCH 16/63] working on windows USB fix --- circup/__init__.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/circup/__init__.py b/circup/__init__.py index faa72a5..d720da0 100644 --- a/circup/__init__.py +++ b/circup/__init__.py @@ -460,6 +460,8 @@ def get_device_versions(self, device_url): connected device. """ # url = urlparse(device_url) + print(f"device_url: {device_url}") + print(f"lib_dir: {self.LIB_DIR_PATH}") return self.get_modules(os.path.join(device_url, self.LIB_DIR_PATH)) def _create_library_directory(self, device_path, library_path): @@ -875,8 +877,8 @@ class USBBackend(Backend): """ def __init__(self): - self.LIB_DIR_PATH = "lib" super().__init__() + self.LIB_DIR_PATH = "lib" def _get_circuitpython_version(self, device_location): """ @@ -894,10 +896,14 @@ def _get_circuitpython_version(self, device_location): :param str device_path: The path to the connected board. :return: A tuple with the version string for CircuitPython and the board ID string. """ + print("device_location:") + print(device_location) device_path = urlparse(device_location).path + print("device_path:") + print(device_path) try: with open( - os.path.join(device_path, "boot_out.txt"), "r", encoding="utf-8" + os.path.join(device_location, "boot_out.txt"), "r", encoding="utf-8" ) as boot: version_line = boot.readline() circuit_python = version_line.split(";")[0].split(" ")[-3] From 168f8ec86d812ad7d1a5e457cef2426dd7fca43b Mon Sep 17 00:00:00 2001 From: foamyguy Date: Wed, 22 Nov 2023 16:58:46 -0600 Subject: [PATCH 17/63] remove prints --- circup/__init__.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/circup/__init__.py b/circup/__init__.py index d720da0..5d03bc6 100644 --- a/circup/__init__.py +++ b/circup/__init__.py @@ -459,9 +459,6 @@ def get_device_versions(self, device_url): :return: A dictionary of metadata about the modules available on the connected device. """ - # url = urlparse(device_url) - print(f"device_url: {device_url}") - print(f"lib_dir: {self.LIB_DIR_PATH}") return self.get_modules(os.path.join(device_url, self.LIB_DIR_PATH)) def _create_library_directory(self, device_path, library_path): @@ -896,11 +893,6 @@ def _get_circuitpython_version(self, device_location): :param str device_path: The path to the connected board. :return: A tuple with the version string for CircuitPython and the board ID string. """ - print("device_location:") - print(device_location) - device_path = urlparse(device_location).path - print("device_path:") - print(device_path) try: with open( os.path.join(device_location, "boot_out.txt"), "r", encoding="utf-8" From a5b40b46d3df3e19a7fd334ea156e1998b6c5dad Mon Sep 17 00:00:00 2001 From: foamyguy Date: Wed, 22 Nov 2023 18:02:48 -0600 Subject: [PATCH 18/63] fixing tests --- tests/mock_device/boot_out.txt | 2 ++ tests/test_circup.py | 56 ++++++++++++++++++++++++---------- 2 files changed, 42 insertions(+), 16 deletions(-) create mode 100644 tests/mock_device/boot_out.txt diff --git a/tests/mock_device/boot_out.txt b/tests/mock_device/boot_out.txt new file mode 100644 index 0000000..ee14d1c --- /dev/null +++ b/tests/mock_device/boot_out.txt @@ -0,0 +1,2 @@ +Adafruit CircuitPython 4.1.0 on 2019-08-02; +Adafruit CircuitPlayground Express with samd21g18 diff --git a/tests/test_circup.py b/tests/test_circup.py index e1ad769..5b2d45c 100644 --- a/tests/test_circup.py +++ b/tests/test_circup.py @@ -389,10 +389,11 @@ def test_Module_update_dir(): m = circup.Module( path, repo, device_version, bundle_version, False, bundle, (None, None) ) + backend = circup.USBBackend() with mock.patch("circup.shutil") as mock_shutil, mock.patch( "circup.os.path.isdir", return_value=True ): - m.update() + backend.update(m) mock_shutil.rmtree.assert_called_once_with(m.path, ignore_errors=True) mock_shutil.copytree.assert_called_once_with(m.bundle_path, m.path) @@ -410,10 +411,11 @@ def test_Module_update_file(): m = circup.Module( path, repo, device_version, bundle_version, False, bundle, (None, None) ) + backend = circup.USBBackend() with mock.patch("circup.shutil") as mock_shutil, mock.patch( "circup.os.remove" ) as mock_remove, mock.patch("circup.os.path.isdir", return_value=False): - m.update() + backend.update(m) mock_remove.assert_called_once_with(m.path) mock_shutil.copyfile.assert_called_once_with(m.bundle_path, m.path) @@ -604,18 +606,21 @@ def test_find_modules(): device_modules = json.load(f) with open("tests/bundle.json", "rb") as f: bundle_modules = json.load(f) + with mock.patch( - "circup.get_device_versions", return_value=device_modules + "circup.USBBackend.get_device_versions", return_value=device_modules ), mock.patch( "circup.get_bundle_versions", return_value=bundle_modules ), mock.patch( "circup.os.path.isfile", return_value=True ): + backend = circup.USBBackend() bundle = circup.Bundle(TEST_BUNDLE_NAME) bundles_list = [bundle] for module in bundle_modules: bundle_modules[module]["bundle"] = bundle - result = circup.find_modules("", bundles_list) + + result = circup.find_modules(backend, "", bundles_list) assert len(result) == 1 assert result[0].name == "adafruit_74hc595" assert ( @@ -630,13 +635,14 @@ def test_find_modules_goes_bang(): and the utility exists with an error code of 1. """ with mock.patch( - "circup.get_device_versions", side_effect=Exception("BANG!") + "circup.USBBackend.get_device_versions", side_effect=Exception("BANG!") ), mock.patch("circup.click") as mock_click, mock.patch( "circup.sys.exit" ) as mock_exit: bundle = circup.Bundle(TEST_BUNDLE_NAME) bundles_list = [bundle] - circup.find_modules("", bundles_list) + backend = circup.USBBackend() + circup.find_modules(backend, "", bundles_list) assert mock_click.echo.call_count == 1 mock_exit.assert_called_once_with(1) @@ -699,7 +705,8 @@ def test_get_circuitpython_version(): "Adafruit CircuitPlayground Express with samd21g18" ) with mock.patch("builtins.open", mock.mock_open(read_data=data_no_id)) as mock_open: - assert circup.get_circuitpython_version(device_path) == ("4.1.0", "") + backend = circup.USBBackend() + assert backend.get_circuitpython_version(device_path) == ("4.1.0", "") mock_open.assert_called_once_with( os.path.join(device_path, "boot_out.txt"), "r", encoding="utf-8" ) @@ -707,7 +714,8 @@ def test_get_circuitpython_version(): with mock.patch( "builtins.open", mock.mock_open(read_data=data_with_id) ) as mock_open: - assert circup.get_circuitpython_version(device_path) == ( + backend = circup.USBBackend() + assert backend.get_circuitpython_version(device_path) == ( "4.1.0", "this_is_a_board", ) @@ -720,8 +728,9 @@ def test_get_device_versions(): """ Ensure get_modules is called with the path for the attached device. """ - with mock.patch("circup.get_modules", return_value="ok") as mock_gm: - assert circup.get_device_versions("TESTDIR") == "ok" + with mock.patch("circup.USBBackend.get_modules", return_value="ok") as mock_gm: + backend = circup.USBBackend() + assert backend.get_device_versions("TESTDIR") == "ok" mock_gm.assert_called_once_with(os.path.join("TESTDIR", "lib")) @@ -730,7 +739,8 @@ def test_get_modules_empty_path(): Sometimes a path to a device or bundle may be empty. Ensure, if this is the case, an empty dictionary is returned. """ - assert circup.get_modules("") == {} + backend = circup.USBBackend() + assert backend.get_modules("") == {} def test_get_modules_that_are_files(): @@ -744,7 +754,8 @@ def test_get_modules_that_are_files(): os.path.join("tests", ".hidden_module.py"), ] with mock.patch("circup.glob.glob", side_effect=[mods, [], []]): - result = circup.get_modules(path) + backend = circup.USBBackend() + result = backend.get_modules(path) assert len(result) == 1 # Hidden files are ignored. assert "local_module" in result assert result["local_module"]["path"] == os.path.join( @@ -767,7 +778,8 @@ def test_get_modules_that_are_directories(): ] mod_files = ["tests/dir_module/my_module.py", "tests/dir_module/__init__.py"] with mock.patch("circup.glob.glob", side_effect=[[], [], mods, mod_files, []]): - result = circup.get_modules(path) + backend = circup.USBBackend() + result = backend.get_modules(path) assert len(result) == 1 assert "dir_module" in result assert result["dir_module"]["path"] == os.path.join("tests", "dir_module", "") @@ -785,7 +797,8 @@ def test_get_modules_that_are_directories_with_no_metadata(): mods = [os.path.join("tests", "bad_module", "")] mod_files = ["tests/bad_module/my_module.py", "tests/bad_module/__init__.py"] with mock.patch("circup.glob.glob", side_effect=[[], [], mods, mod_files, []]): - result = circup.get_modules(path) + backend = circup.USBBackend() + result = backend.get_modules(path) assert len(result) == 1 assert "bad_module" in result assert result["bad_module"]["path"] == os.path.join("tests", "bad_module", "") @@ -1019,7 +1032,8 @@ def test_libraries_from_imports(): "adafruit_touchscreen", ] test_file = str(pathlib.Path(__file__).parent / "import_styles.py") - result = circup.libraries_from_imports(None, test_file, mod_names) + backend = circup.USBBackend() + result = backend.libraries_from_imports(test_file, mod_names) print(result) assert result == [ "adafruit_bus_device", @@ -1033,6 +1047,16 @@ def test_libraries_from_imports_bad(): """Ensure that we catch an import error""" TEST_BUNDLE_MODULES = {"one.py": {}, "two.py": {}, "three.py": {}} runner = CliRunner() + with mock.patch("circup.get_bundle_versions", return_value=TEST_BUNDLE_MODULES): - result = runner.invoke(circup.install, ["--auto-file", "./tests/bad_python.py"]) + result = runner.invoke( + circup.main, + [ + "--path", + "./tests/mock_device/", + "install", + "--auto-file", + "./tests/bad_python.py", + ], + ) assert result.exit_code == 2 From 75e501204ac7a3286f666d5965a751c2269fc6da Mon Sep 17 00:00:00 2001 From: foamyguy Date: Wed, 22 Nov 2023 18:06:07 -0600 Subject: [PATCH 19/63] license for mock boot_out --- tests/mock_device/boot_out.txt.license | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 tests/mock_device/boot_out.txt.license diff --git a/tests/mock_device/boot_out.txt.license b/tests/mock_device/boot_out.txt.license new file mode 100644 index 0000000..7207a5d --- /dev/null +++ b/tests/mock_device/boot_out.txt.license @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: 2023 Tim Cocks, written for Adafruit Industries +# +# SPDX-License-Identifier: MIT From 51f9670610aafa52d4eacb95df1f313d432448c9 Mon Sep 17 00:00:00 2001 From: foamyguy Date: Mon, 27 Nov 2023 09:50:17 -0600 Subject: [PATCH 20/63] update readme for web workflow support --- README.rst | 46 ++++++++++++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/README.rst b/README.rst index 5a23414..8439e50 100644 --- a/README.rst +++ b/README.rst @@ -89,24 +89,32 @@ To get help, just type the command:: A tool to manage and update libraries on a CircuitPython device. Options: - --verbose Comprehensive logging is sent to stdout. - --version Show the version and exit. - --path DIRECTORY Path to CircuitPython directory. Overrides automatic - path detection. - --help Show this message and exit. - -r --requirement Supports requirements.txt tracking of library - requirements with freeze and install commands. + --verbose Comprehensive logging is sent to stdout. + --path DIRECTORY Path to CircuitPython directory. Overrides automatic + path detection. + --host TEXT Hostname or IP address of a device. Overrides automatic + path detection. + --password TEXT Password to use for authentication when --host is used. + --board-id TEXT Manual Board ID of the CircuitPython device. If provided + in combination with --cpy-version, it overrides the + detected board ID. + --cpy-version TEXT Manual CircuitPython version. If provided in combination + with --board-id, it overrides the detected CPy version. + --version Show the version and exit. + --help Show this message and exit. Commands: - freeze Output details of all the modules found on the connected... - install Installs .mpy version of named module(s) onto the device. - install --py Installs .py version of named module(s). - list Lists all out of date modules found on the connected... - show Show the long list of all available modules in the bundle. - show Search the names in the modules in the bundle for a match. - uninstall Uninstall a named module(s) from the connected device. - update Update modules on the device. Use --all to automatically update - all modules. + bundle-add Add bundles to the local bundles list, by "user/repo"... + bundle-remove Remove one or more bundles from the local bundles list. + bundle-show Show the list of bundles, default and local, with URL,... + freeze Output details of all the modules found on the connected... + install Install a named module(s) onto the device. + list Lists all out of date modules found on the connected... + show Show a list of available modules in the bundle. + uninstall Uninstall a named module(s) from the connected device. + update Update modules on the device. Use --all to automatically + update all modules without Major Version warnings. + To automatically install all modules imported by ``code.py``, @@ -221,6 +229,12 @@ The ``--version`` flag will tell you the current version of the $ circup --version CircUp, A CircuitPython module updater. Version 0.0.1 + +To use circup via the `Web Workflow `_. on devices that support it. Use the ``--host`` and ``--password`` arguments before your circup command.`:: + + $ circup --host 192.168.1.119 --password s3cr3t install adafruit_hid + $ circup --host cpy-9573b2.local --password s3cr3t install adafruit_hid + That's it! From 467740088cce30c8c403213ea1dada3f24870cb5 Mon Sep 17 00:00:00 2001 From: foamyguy Date: Mon, 27 Nov 2023 09:51:09 -0600 Subject: [PATCH 21/63] remove extra tick mark --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 8439e50..118cb39 100644 --- a/README.rst +++ b/README.rst @@ -230,7 +230,7 @@ The ``--version`` flag will tell you the current version of the CircUp, A CircuitPython module updater. Version 0.0.1 -To use circup via the `Web Workflow `_. on devices that support it. Use the ``--host`` and ``--password`` arguments before your circup command.`:: +To use circup via the `Web Workflow `_. on devices that support it. Use the ``--host`` and ``--password`` arguments before your circup command.:: $ circup --host 192.168.1.119 --password s3cr3t install adafruit_hid $ circup --host cpy-9573b2.local --password s3cr3t install adafruit_hid From d3eda67ddb61d4c343b11fd07c660407f9ada7d2 Mon Sep 17 00:00:00 2001 From: foamyguy Date: Wed, 29 Nov 2023 07:21:44 -0600 Subject: [PATCH 22/63] fix typo, change name to DiskBackend, remove wrapper _get_circuitpython_version --- circup/__init__.py | 22 ++++++++++------------ tests/test_circup.py | 24 ++++++++++++------------ 2 files changed, 22 insertions(+), 24 deletions(-) diff --git a/circup/__init__.py b/circup/__init__.py index 5d03bc6..4075465 100644 --- a/circup/__init__.py +++ b/circup/__init__.py @@ -49,7 +49,7 @@ LOG_DIR = appdirs.user_log_dir(appname="circup", appauthor="adafruit") #: The location of the log file for the utility. LOGFILE = os.path.join(LOG_DIR, "circup.log") -#: The localtion to store a local copy of code.py for use with --auto and +#: The location to store a local copy of code.py for use with --auto and # web workflow LOCAL_CODE_PY_COPY = os.path.join(DATA_DIR, "code.tmp.py") #: The libraries (and blank lines) which don't go on devices @@ -419,21 +419,17 @@ class Backend: def __init__(self): self.LIB_DIR_PATH = None - def _get_circuitpython_version(self, device_location): - """ - To be overridden by subclass - """ - raise NotImplementedError - def get_circuitpython_version(self, device_location): """ + Must be overridden by subclass for implementation! + Returns the version number of CircuitPython running on the board connected via ``device_url``, along with the board ID. :param str device_location: http based device URL or local file path. :return: A tuple with the version string for CircuitPython and the board ID string. """ - return self._get_circuitpython_version(device_location) + raise NotImplementedError def _get_modules(self, device_lib_path): """ @@ -578,6 +574,7 @@ def install_file_http(self, source, target): """ url = urlparse(target) auth = HTTPBasicAuth("", url.password) + print(f"target: {target}") with open(source, "rb") as fp: r = requests.put(target, fp.read(), auth=auth) @@ -591,6 +588,7 @@ def install_dir_http(self, source, target): """ url = urlparse(target) auth = HTTPBasicAuth("", url.password) + print(f"target: {target}") # Create the top level directory. r = requests.put(target + ("/" if not target.endswith("/") else ""), auth=auth) @@ -611,7 +609,7 @@ def install_dir_http(self, source, target): r = requests.put(target + rel_path + "/" + name, auth=auth) r.raise_for_status() - def _get_circuitpython_version(self, url): + def get_circuitpython_version(self, url): """ Returns the version number of CircuitPython running on the board connected via ``device_path``, along with the board ID. This is obtained using @@ -868,7 +866,7 @@ def _get_modules_file(path): return result -class USBBackend(Backend): +class DiskBackend(Backend): """ Backend for interacting with a device via USB Workflow """ @@ -877,7 +875,7 @@ def __init__(self): super().__init__() self.LIB_DIR_PATH = "lib" - def _get_circuitpython_version(self, device_location): + def get_circuitpython_version(self, device_location): """ Returns the version number of CircuitPython running on the board connected via ``device_path``, along with the board ID. This is obtained from the @@ -1663,7 +1661,7 @@ def main(ctx, verbose, path, host, password, board_id, cpy_version): # pragma: if using_webworkflow: ctx.obj["backend"] = WebBackend(host=host, password=password) else: - ctx.obj["backend"] = USBBackend() + ctx.obj["backend"] = DiskBackend() if verbose: # Configure additional logging to stdout. diff --git a/tests/test_circup.py b/tests/test_circup.py index 5b2d45c..f34e651 100644 --- a/tests/test_circup.py +++ b/tests/test_circup.py @@ -389,7 +389,7 @@ def test_Module_update_dir(): m = circup.Module( path, repo, device_version, bundle_version, False, bundle, (None, None) ) - backend = circup.USBBackend() + backend = circup.DiskBackend() with mock.patch("circup.shutil") as mock_shutil, mock.patch( "circup.os.path.isdir", return_value=True ): @@ -411,7 +411,7 @@ def test_Module_update_file(): m = circup.Module( path, repo, device_version, bundle_version, False, bundle, (None, None) ) - backend = circup.USBBackend() + backend = circup.DiskBackend() with mock.patch("circup.shutil") as mock_shutil, mock.patch( "circup.os.remove" ) as mock_remove, mock.patch("circup.os.path.isdir", return_value=False): @@ -614,7 +614,7 @@ def test_find_modules(): ), mock.patch( "circup.os.path.isfile", return_value=True ): - backend = circup.USBBackend() + backend = circup.DiskBackend() bundle = circup.Bundle(TEST_BUNDLE_NAME) bundles_list = [bundle] for module in bundle_modules: @@ -641,7 +641,7 @@ def test_find_modules_goes_bang(): ) as mock_exit: bundle = circup.Bundle(TEST_BUNDLE_NAME) bundles_list = [bundle] - backend = circup.USBBackend() + backend = circup.DiskBackend() circup.find_modules(backend, "", bundles_list) assert mock_click.echo.call_count == 1 mock_exit.assert_called_once_with(1) @@ -705,7 +705,7 @@ def test_get_circuitpython_version(): "Adafruit CircuitPlayground Express with samd21g18" ) with mock.patch("builtins.open", mock.mock_open(read_data=data_no_id)) as mock_open: - backend = circup.USBBackend() + backend = circup.DiskBackend() assert backend.get_circuitpython_version(device_path) == ("4.1.0", "") mock_open.assert_called_once_with( os.path.join(device_path, "boot_out.txt"), "r", encoding="utf-8" @@ -714,7 +714,7 @@ def test_get_circuitpython_version(): with mock.patch( "builtins.open", mock.mock_open(read_data=data_with_id) ) as mock_open: - backend = circup.USBBackend() + backend = circup.DiskBackend() assert backend.get_circuitpython_version(device_path) == ( "4.1.0", "this_is_a_board", @@ -729,7 +729,7 @@ def test_get_device_versions(): Ensure get_modules is called with the path for the attached device. """ with mock.patch("circup.USBBackend.get_modules", return_value="ok") as mock_gm: - backend = circup.USBBackend() + backend = circup.DiskBackend() assert backend.get_device_versions("TESTDIR") == "ok" mock_gm.assert_called_once_with(os.path.join("TESTDIR", "lib")) @@ -739,7 +739,7 @@ def test_get_modules_empty_path(): Sometimes a path to a device or bundle may be empty. Ensure, if this is the case, an empty dictionary is returned. """ - backend = circup.USBBackend() + backend = circup.DiskBackend() assert backend.get_modules("") == {} @@ -754,7 +754,7 @@ def test_get_modules_that_are_files(): os.path.join("tests", ".hidden_module.py"), ] with mock.patch("circup.glob.glob", side_effect=[mods, [], []]): - backend = circup.USBBackend() + backend = circup.DiskBackend() result = backend.get_modules(path) assert len(result) == 1 # Hidden files are ignored. assert "local_module" in result @@ -778,7 +778,7 @@ def test_get_modules_that_are_directories(): ] mod_files = ["tests/dir_module/my_module.py", "tests/dir_module/__init__.py"] with mock.patch("circup.glob.glob", side_effect=[[], [], mods, mod_files, []]): - backend = circup.USBBackend() + backend = circup.DiskBackend() result = backend.get_modules(path) assert len(result) == 1 assert "dir_module" in result @@ -797,7 +797,7 @@ def test_get_modules_that_are_directories_with_no_metadata(): mods = [os.path.join("tests", "bad_module", "")] mod_files = ["tests/bad_module/my_module.py", "tests/bad_module/__init__.py"] with mock.patch("circup.glob.glob", side_effect=[[], [], mods, mod_files, []]): - backend = circup.USBBackend() + backend = circup.DiskBackend() result = backend.get_modules(path) assert len(result) == 1 assert "bad_module" in result @@ -1032,7 +1032,7 @@ def test_libraries_from_imports(): "adafruit_touchscreen", ] test_file = str(pathlib.Path(__file__).parent / "import_styles.py") - backend = circup.USBBackend() + backend = circup.DiskBackend() result = backend.libraries_from_imports(test_file, mod_names) print(result) assert result == [ From fb6cd847a87202e4c1a027f46412318611021fbd Mon Sep 17 00:00:00 2001 From: foamyguy Date: Wed, 29 Nov 2023 07:55:44 -0600 Subject: [PATCH 23/63] starting move backends to own file --- circup/__init__.py | 698 +-------------------------------------------- circup/backends.py | 582 +++++++++++++++++++++++++++++++++++++ circup/shared.py | 127 +++++++++ 3 files changed, 723 insertions(+), 684 deletions(-) create mode 100644 circup/backends.py create mode 100644 circup/shared.py diff --git a/circup/__init__.py b/circup/__init__.py index 4075465..2516dba 100644 --- a/circup/__init__.py +++ b/circup/__init__.py @@ -30,11 +30,13 @@ from requests.auth import HTTPBasicAuth from semver import VersionInfo +from circup.shared import DATA_DIR, BAD_FILE_FORMAT, extract_metadata, CPY_VERSION, _get_modules_file +from circup.backends import WebBackend, DiskBackend + # Useful constants. #: Flag to indicate if the command is being run in verbose mode. VERBOSE = False -#: The location of data files used by circup (following OS conventions). -DATA_DIR = appdirs.user_data_dir(appname="circup", appauthor="adafruit") + #: The path to the JSON file containing the metadata about the bundles. BUNDLE_CONFIG_FILE = pkg_resources.resource_filename( "circup", "config/bundle_config.json" @@ -49,9 +51,7 @@ LOG_DIR = appdirs.user_log_dir(appname="circup", appauthor="adafruit") #: The location of the log file for the utility. LOGFILE = os.path.join(LOG_DIR, "circup.log") -#: The location to store a local copy of code.py for use with --auto and -# web workflow -LOCAL_CODE_PY_COPY = os.path.join(DATA_DIR, "code.tmp.py") + #: The libraries (and blank lines) which don't go on devices NOT_MCU_LIBRARIES = [ "", @@ -62,14 +62,12 @@ "circuitpython_typing", "pyserial", ] -#: The version of CircuitPython found on the connected device. -CPY_VERSION = "" + #: Module formats list (and the other form used in github files) PLATFORMS = {"py": "py", "8mpy": "8.x-mpy", "9mpy": "9.x-mpy"} #: Commands that do not require an attached board BOARDLESS_COMMANDS = ["show", "bundle-add", "bundle-remove", "bundle-show"] -#: Version identifier for a bad MPY file format -BAD_FILE_FORMAT = "Invalid" + #: Timeout for requests calls like get() REQUESTS_TIMEOUT = 30 @@ -129,6 +127,8 @@ def lib_dir(self, platform): :return: The path to the lib directory for the platform. """ tag = self.current_tag + print(platform) + print(PLATFORMS) return os.path.join( self.dir.format(platform=platform), self.basename.format(platform=PLATFORMS[platform], tag=tag), @@ -410,612 +410,11 @@ def __repr__(self): ) -class Backend: - """ - Backend parent class to be extended for workflow specific - implementations - """ - - def __init__(self): - self.LIB_DIR_PATH = None - - def get_circuitpython_version(self, device_location): - """ - Must be overridden by subclass for implementation! - - Returns the version number of CircuitPython running on the board connected - via ``device_url``, along with the board ID. - - :param str device_location: http based device URL or local file path. - :return: A tuple with the version string for CircuitPython and the board ID string. - """ - raise NotImplementedError - - def _get_modules(self, device_lib_path): - """ - To be overridden by subclass - """ - raise NotImplementedError - - def get_modules(self, device_url): - """ - Get a dictionary containing metadata about all the Python modules found in - the referenced path. - - :param str device_url: URL to be used to find modules. - :return: A dictionary containing metadata about the found modules. - """ - return self._get_modules(device_url) - - def get_device_versions(self, device_url): - """ - Returns a dictionary of metadata from modules on the connected device. - - :param str device_url: URL for the device. - :return: A dictionary of metadata about the modules available on the - connected device. - """ - return self.get_modules(os.path.join(device_url, self.LIB_DIR_PATH)) - - def _create_library_directory(self, device_path, library_path): - """ - To be overridden by subclass - """ - raise NotImplementedError - - def _install_module_py(self, library_path, metadata): - """ - To be overridden by subclass - """ - raise NotImplementedError - - def _install_module_mpy(self, bundle, library_path, metadata): - """ - To be overridden by subclass - """ - raise NotImplementedError - - # pylint: disable=too-many-locals,too-many-branches,too-many-arguments - def install_module( - self, device_path, device_modules, name, pyext, mod_names - ): # pragma: no cover - """ - Finds a connected device and installs a given module name if it - is available in the current module bundle and is not already - installed on the device. - TODO: There is currently no check for the version. - - :param str device_path: The path to the connected board. - :param list(dict) device_modules: List of module metadata from the device. - :param str name: Name of module to install - :param bool pyext: Boolean to specify if the module should be installed from - source or from a pre-compiled module - :param mod_names: Dictionary of metadata from modules that can be generated - with get_bundle_versions() - """ - if not name: - click.echo("No module name(s) provided.") - elif name in mod_names: - # Grab device modules to check if module already installed - if name in device_modules: - click.echo("'{}' is already installed.".format(name)) - return - - library_path = os.path.join(device_path, self.LIB_DIR_PATH) - - # Create the library directory first. - self._create_library_directory(device_path, library_path) - - metadata = mod_names[name] - bundle = metadata["bundle"] - if pyext: - # Use Python source for module. - self._install_module_py(library_path, metadata) - else: - # Use pre-compiled mpy modules. - self._install_module_mpy(bundle, library_path, metadata) - click.echo("Installed '{}'.".format(name)) - else: - click.echo("Unknown module named, '{}'.".format(name)) - - def libraries_from_imports(self, code_py, mod_names): - """ - To be overridden by subclass - """ - raise NotImplementedError - - def uninstall(self, device_path, module_path): - """ - To be overridden by subclass - """ - raise NotImplementedError - - def update(self, module): - """ - To be overridden by subclass - """ - raise NotImplementedError - - def libraries_from_code_py(self, code_py, mod_names): - """ - Parse the given code.py file and return the imported libraries - - :param str code_py: Full path of the code.py file - :return: sequence of library names - """ - # pylint: disable=broad-except - try: - found_imports = findimports.find_imports(code_py) - except Exception as ex: # broad exception because anything could go wrong - logger.exception(ex) - click.secho('Unable to read the auto file: "{}"'.format(str(ex)), fg="red") - sys.exit(2) - # pylint: enable=broad-except - imports = [info.name.split(".", 1)[0] for info in found_imports] - return [r for r in imports if r in mod_names] - - -class WebBackend(Backend): - """ - Backend for interacting with a device via Web Workflow - """ - - def __init__(self, host, password): - super().__init__() - self.LIB_DIR_PATH = "fs/lib/" - self.host = host - self.password = password - - def install_file_http(self, source, target): - """ - Install file to device using web workflow. - :param source source file. - :param target destination URL. Should have password embedded. - """ - url = urlparse(target) - auth = HTTPBasicAuth("", url.password) - print(f"target: {target}") - - with open(source, "rb") as fp: - r = requests.put(target, fp.read(), auth=auth) - r.raise_for_status() - - def install_dir_http(self, source, target): - """ - Install directory to device using web workflow. - :param source source directory. - :param target destination URL. Should have password embedded. - """ - url = urlparse(target) - auth = HTTPBasicAuth("", url.password) - print(f"target: {target}") - - # Create the top level directory. - r = requests.put(target + ("/" if not target.endswith("/") else ""), auth=auth) - r.raise_for_status() - - # Traverse the directory structure and create the directories/files. - for root, dirs, files in os.walk(source): - rel_path = os.path.relpath(root, source) - if rel_path == ".": - rel_path = "" - for name in files: - with open(os.path.join(root, name), "rb") as fp: - r = requests.put( - target + rel_path + "/" + name, fp.read(), auth=auth - ) - r.raise_for_status() - for name in dirs: - r = requests.put(target + rel_path + "/" + name, auth=auth) - r.raise_for_status() - - def get_circuitpython_version(self, url): - """ - Returns the version number of CircuitPython running on the board connected - via ``device_path``, along with the board ID. This is obtained using - RESTful API from the /cp/version.json URL. - - :param str url: board URL. - :return: A tuple with the version string for CircuitPython and the board ID string. - """ - # pylint: disable=arguments-renamed - r = requests.get(url + "/cp/version.json") - # pylint: disable=no-member - if r.status_code != requests.codes.ok: - click.secho( - f" Unable to get version from {url}: {r.status_code}", fg="red" - ) - sys.exit(1) - # pylint: enable=no-member - ver_json = r.json() - return ver_json.get("version"), ver_json.get("board_id") - - def _get_modules(self, device_lib_path): - return self._get_modules_http(device_lib_path) - - def _get_modules_http(self, url): - """ - Get a dictionary containing metadata about all the Python modules found using - the referenced URL. - - :param str url: URL for the modules. - :return: A dictionary containing metadata about the found modules. - """ - result = {} - u = urlparse(url) - auth = HTTPBasicAuth("", u.password) - r = requests.get(url, auth=auth, headers={"Accept": "application/json"}) - r.raise_for_status() - - directory_mods = [] - single_file_mods = [] - for entry in r.json(): - entry_name = entry.get("name") - if entry.get("directory"): - directory_mods.append(entry_name) - else: - if entry_name.endswith(".py") or entry_name.endswith(".mpy"): - single_file_mods.append(entry_name) - - self._get_modules_http_single_mods(auth, result, single_file_mods, url) - self._get_modules_http_dir_mods(auth, directory_mods, result, url) - - return result - - def _get_modules_http_dir_mods(self, auth, directory_mods, result, url): - """ - #TODO describe what this does - - :param auth HTTP authentication. - :param directory_mods list of modules. - :param result dictionary for the result. - :param url: URL of the device. - """ - for dm in directory_mods: - dm_url = url + dm + "/" - r = requests.get(dm_url, auth=auth, headers={"Accept": "application/json"}) - r.raise_for_status() - mpy = False - for entry in r.json(): - entry_name = entry.get("name") - if not entry.get("directory") and ( - entry_name.endswith(".py") or entry_name.endswith(".mpy") - ): - if entry_name.endswith(".mpy"): - mpy = True - r = requests.get(dm_url + entry_name, auth=auth) - r.raise_for_status() - idx = entry_name.rfind(".") - with tempfile.NamedTemporaryFile( - prefix=entry_name[:idx] + "-", - suffix=entry_name[idx:], - delete=False, - ) as fp: - fp.write(r.content) - tmp_name = fp.name - metadata = extract_metadata(tmp_name) - os.remove(tmp_name) - if "__version__" in metadata: - metadata["path"] = dm_url - result[dm] = metadata - # break now if any of the submodules has a bad format - if metadata["__version__"] == BAD_FILE_FORMAT: - break - - if result.get(dm) is None: - result[dm] = {"path": dm_url, "mpy": mpy} - - def _get_modules_http_single_mods(self, auth, result, single_file_mods, url): - """ - :param auth HTTP authentication. - :param single_file_mods list of modules. - :param result dictionary for the result. - :param url: URL of the device. - """ - for sfm in single_file_mods: - sfm_url = url + sfm - r = requests.get(sfm_url, auth=auth) - r.raise_for_status() - idx = sfm.rfind(".") - with tempfile.NamedTemporaryFile( - prefix=sfm[:idx] + "-", suffix=sfm[idx:], delete=False - ) as fp: - fp.write(r.content) - tmp_name = fp.name - metadata = extract_metadata(tmp_name) - os.remove(tmp_name) - metadata["path"] = sfm_url - result[sfm[:idx]] = metadata - - def _create_library_directory(self, device_path, library_path): - url = urlparse(device_path) - auth = HTTPBasicAuth("", url.password) - r = requests.put(library_path, auth=auth) - r.raise_for_status() - - def _install_module_mpy(self, bundle, library_path, metadata): - """ - :param bundle library bundle. - :param library_path library path - :param metadata dictionary. - """ - module_name = os.path.basename(metadata["path"]).replace(".py", ".mpy") - if not module_name: - # Must be a directory based module. - module_name = os.path.basename(os.path.dirname(metadata["path"])) - major_version = CPY_VERSION.split(".")[0] - bundle_platform = "{}mpy".format(major_version) - bundle_path = os.path.join(bundle.lib_dir(bundle_platform), module_name) - if os.path.isdir(bundle_path): - - self.install_dir_http(bundle_path, library_path + module_name) - - elif os.path.isfile(bundle_path): - target = os.path.basename(bundle_path) - - self.install_file_http(bundle_path, library_path + target) - - else: - raise IOError("Cannot find compiled version of module.") - - # pylint: enable=too-many-locals,too-many-branches - def _install_module_py(self, library_path, metadata): - """ - :param library_path library path - :param metadata dictionary. - """ - source_path = metadata["path"] # Path to Python source version. - if os.path.isdir(source_path): - target = os.path.basename(os.path.dirname(source_path)) - self.install_dir_http(source_path, library_path + target) - - else: - target = os.path.basename(source_path) - self.install_file_http(source_path, library_path + target) - - def libraries_from_imports(self, code_py, mod_names): - """ - Parse the given code.py file and return the imported libraries - - :param str code_py: Full path of the code.py file - :return: sequence of library names - """ - url = code_py - auth = HTTPBasicAuth("", self.password) - r = requests.get(url, auth=auth) - r.raise_for_status() - with open(LOCAL_CODE_PY_COPY, "w", encoding="utf-8") as f: - f.write(r.text) - code_py = LOCAL_CODE_PY_COPY - - return self.libraries_from_code_py(code_py, mod_names) - - def uninstall(self, device_path, module_path): - """ - Uninstall given module on device using REST API. - """ - url = urlparse(device_path) - auth = HTTPBasicAuth("", url.password) - r = requests.delete(module_path, auth=auth) - r.raise_for_status() - - def update(self, module): - """ - Delete the module on the device, then copy the module from the bundle - back onto the device. - - The caller is expected to handle any exceptions raised. - """ - self._update_http(module) - - def _update_http(self, module): - """ - Update the module using web workflow. - """ - if module.file: - # Copy the file (will overwrite). - self.install_file_http(module.bundle_path, module.path) - else: - # Delete the directory (recursive) first. - url = urlparse(module.path) - auth = HTTPBasicAuth("", url.password) - r = requests.delete(module.path, auth=auth) - r.raise_for_status() - self.install_dir_http(module.bundle_path, module.path) - - -def _get_modules_file(path): - """ - Get a dictionary containing metadata about all the Python modules found in - the referenced file system path. - - :param str path: The directory in which to find modules. - :return: A dictionary containing metadata about the found modules. - """ - result = {} - if not path: - return result - single_file_py_mods = glob.glob(os.path.join(path, "*.py")) - single_file_mpy_mods = glob.glob(os.path.join(path, "*.mpy")) - package_dir_mods = [ - d - for d in glob.glob(os.path.join(path, "*", "")) - if not os.path.basename(os.path.normpath(d)).startswith(".") - ] - single_file_mods = single_file_py_mods + single_file_mpy_mods - for sfm in [f for f in single_file_mods if not os.path.basename(f).startswith(".")]: - metadata = extract_metadata(sfm) - metadata["path"] = sfm - result[os.path.basename(sfm).replace(".py", "").replace(".mpy", "")] = metadata - for package_path in package_dir_mods: - name = os.path.basename(os.path.dirname(package_path)) - py_files = glob.glob(os.path.join(package_path, "**/*.py"), recursive=True) - mpy_files = glob.glob(os.path.join(package_path, "**/*.mpy"), recursive=True) - all_files = py_files + mpy_files - # default value - result[name] = {"path": package_path, "mpy": bool(mpy_files)} - # explore all the submodules to detect bad ones - for source in [f for f in all_files if not os.path.basename(f).startswith(".")]: - metadata = extract_metadata(source) - if "__version__" in metadata: - metadata["path"] = package_path - result[name] = metadata - # break now if any of the submodules has a bad format - if metadata["__version__"] == BAD_FILE_FORMAT: - break - return result - -class DiskBackend(Backend): - """ - Backend for interacting with a device via USB Workflow - """ - def __init__(self): - super().__init__() - self.LIB_DIR_PATH = "lib" - def get_circuitpython_version(self, device_location): - """ - Returns the version number of CircuitPython running on the board connected - via ``device_path``, along with the board ID. This is obtained from the - ``boot_out.txt`` file on the device, whose first line will start with - something like this:: - Adafruit CircuitPython 4.1.0 on 2019-08-02; - While the second line is:: - - Board ID:raspberry_pi_pico - - :param str device_path: The path to the connected board. - :return: A tuple with the version string for CircuitPython and the board ID string. - """ - try: - with open( - os.path.join(device_location, "boot_out.txt"), "r", encoding="utf-8" - ) as boot: - version_line = boot.readline() - circuit_python = version_line.split(";")[0].split(" ")[-3] - board_line = boot.readline() - if board_line.startswith("Board ID:"): - board_id = board_line[9:].strip() - else: - board_id = "" - except FileNotFoundError: - click.secho( - "Missing file boot_out.txt on the device: wrong path or drive corrupted.", - fg="red", - ) - logger.error("boot_out.txt not found.") - sys.exit(1) - - return circuit_python, board_id - - def _get_modules(self, device_lib_path): - """ - Get a dictionary containing metadata about all the Python modules found in - the referenced path. - - :param str device_lib_path: URL to be used to find modules. - :return: A dictionary containing metadata about the found modules. - """ - return _get_modules_file(device_lib_path) - - def _create_library_directory(self, device_path, library_path): - if not os.path.exists(library_path): # pragma: no cover - os.makedirs(library_path) - - def _install_module_mpy(self, bundle, library_path, metadata): - """ - :param bundle library bundle. - :param library_path library path - :param metadata dictionary. - """ - module_name = os.path.basename(metadata["path"]).replace(".py", ".mpy") - if not module_name: - # Must be a directory based module. - module_name = os.path.basename(os.path.dirname(metadata["path"])) - major_version = CPY_VERSION.split(".")[0] - bundle_platform = "{}mpy".format(major_version) - bundle_path = os.path.join(bundle.lib_dir(bundle_platform), module_name) - if os.path.isdir(bundle_path): - target_path = os.path.join(library_path, module_name) - # Copy the directory. - shutil.copytree(bundle_path, target_path) - elif os.path.isfile(bundle_path): - target = os.path.basename(bundle_path) - target_path = os.path.join(library_path, target) - # Copy file. - shutil.copyfile(bundle_path, target_path) - else: - raise IOError("Cannot find compiled version of module.") - - # pylint: enable=too-many-locals,too-many-branches - def _install_module_py(self, library_path, metadata): - """ - :param library_path library path - :param metadata dictionary. - """ - - source_path = metadata["path"] # Path to Python source version. - if os.path.isdir(source_path): - target = os.path.basename(os.path.dirname(source_path)) - target_path = os.path.join(library_path, target) - # Copy the directory. - shutil.copytree(source_path, target_path) - else: - target = os.path.basename(source_path) - target_path = os.path.join(library_path, target) - # Copy file. - shutil.copyfile(source_path, target_path) - - def libraries_from_imports(self, code_py, mod_names): - """ - Parse the given code.py file and return the imported libraries - - :param str code_py: Full path of the code.py file - :return: sequence of library names - """ - return self.libraries_from_code_py(code_py, mod_names) - - def uninstall(self, device_path, module_path): - """ - Uninstall module using local file system. - """ - library_path = os.path.join(device_path, "lib") - if os.path.isdir(module_path): - target = os.path.basename(os.path.dirname(module_path)) - target_path = os.path.join(library_path, target) - # Remove the directory. - shutil.rmtree(target_path) - else: - target = os.path.basename(module_path) - target_path = os.path.join(library_path, target) - # Remove file - os.remove(target_path) - - def update(self, module): - """ - Delete the module on the device, then copy the module from the bundle - back onto the device. - - The caller is expected to handle any exceptions raised. - """ - self._update_file(module) - - def _update_file(self, module): - """ - Update the module using file system. - """ - if os.path.isdir(module.path): - # Delete and copy the directory. - shutil.rmtree(module.path, ignore_errors=True) - shutil.copytree(module.bundle_path, module.path) - else: - # Delete and copy file. - os.remove(module.path) - shutil.copyfile(module.bundle_path, module.path) def clean_library_name(assumed_library_name): @@ -1106,76 +505,7 @@ def ensure_latest_bundle(bundle): logger.info("Current bundle up to date %s.", tag) -def extract_metadata(path): - """ - Given a file path, return a dictionary containing metadata extracted from - dunder attributes found therein. Works with both .py and .mpy files. - - For Python source files, such metadata assignments should be simple and - single-line. For example:: - - __version__ = "1.1.4" - __repo__ = "https://github.com/adafruit/SomeLibrary.git" - For byte compiled .mpy files, a brute force / backtrack approach is used - to find the __version__ number in the file -- see comments in the - code for the implementation details. - - :param str path: The path to the file containing the metadata. - :return: The dunder based metadata found in the file, as a dictionary. - """ - result = {} - logger.info("%s", path) - if path.endswith(".py"): - result["mpy"] = False - with open(path, "r", encoding="utf-8") as source_file: - content = source_file.read() - #: The regex used to extract ``__version__`` and ``__repo__`` assignments. - dunder_key_val = r"""(__\w+__)(?:\s*:\s*\w+)?\s*=\s*(?:['"]|\(\s)(.+)['"]""" - for match in re.findall(dunder_key_val, content): - result[match[0]] = str(match[1]) - elif path.endswith(".mpy"): - result["mpy"] = True - with open(path, "rb") as mpy_file: - content = mpy_file.read() - # Track the MPY version number - mpy_version = content[0:2] - compatibility = None - loc = -1 - # Find the start location of the __version__ - if mpy_version == b"M\x03": - # One byte for the length of "__version__" - loc = content.find(b"__version__") - 1 - compatibility = (None, "7.0.0-alpha.1") - elif mpy_version == b"C\x05": - # Two bytes in mpy version 5 - loc = content.find(b"__version__") - 2 - compatibility = ("7.0.0-alpha.1", None) - if loc > -1: - # Backtrack until a byte value of the offset is reached. - offset = 1 - while offset < loc: - val = int(content[loc - offset]) - if mpy_version == b"C\x05": - val = val // 2 - if val == offset - 1: # Off by one..! - # Found version, extract the number given boundaries. - start = loc - offset + 1 # No need for prepended length. - end = loc # Up to the start of the __version__. - version = content[start:end] # Slice the version number. - # Create a string version as metadata in the result. - result["__version__"] = version.decode("utf-8") - break # Nothing more to do. - offset += 1 # ...and again but backtrack by one. - if compatibility: - result["compatibility"] = compatibility - else: - # not a valid MPY file - result["__version__"] = BAD_FILE_FORMAT - - if result: - logger.info("Extracted metadata: %s", result) - return result def find_device(): @@ -1346,7 +676,7 @@ def get_bundle_versions(bundles_list, avoid_download=False): if not avoid_download or not os.path.isdir(bundle.lib_dir("py")): ensure_latest_bundle(bundle) path = bundle.lib_dir("py") - path_modules = _get_modules_file(path) + path_modules = _get_modules_file(path, logger) for name, module in path_modules.items(): module["bundle"] = bundle if name not in all_the_modules: # here we decide the order of priority @@ -1654,14 +984,14 @@ def main(ctx, verbose, path, host, password, board_id, cpy_version): # pragma: A tool to manage and update libraries on a CircuitPython device. """ ctx.ensure_object(dict) - + device_path = get_device_path(host, password, path) using_webworkflow = "host" in ctx.params.keys() and ctx.params["host"] is not None ctx.obj["using_webworkflow"] = using_webworkflow if using_webworkflow: - ctx.obj["backend"] = WebBackend(host=host, password=password) + ctx.obj["backend"] = WebBackend(host=host, password=password, logger=logger) else: - ctx.obj["backend"] = DiskBackend() + ctx.obj["backend"] = DiskBackend(device_path, logger) if verbose: # Configure additional logging to stdout. @@ -1684,7 +1014,7 @@ def main(ctx, verbose, path, host, password, board_id, cpy_version): # pragma: if ctx.invoked_subcommand in BOARDLESS_COMMANDS: return - device_path = get_device_path(host, password, path) + ctx.obj["DEVICE_PATH"] = device_path latest_version = get_latest_release_from_url( "https://github.com/adafruit/circuitpython/releases/latest" diff --git a/circup/backends.py b/circup/backends.py new file mode 100644 index 0000000..9edfc5e --- /dev/null +++ b/circup/backends.py @@ -0,0 +1,582 @@ +import os +import shutil +import sys +import tempfile +from urllib.parse import urlparse + +import click +import findimports +import requests +from requests.auth import HTTPBasicAuth + +from circup.shared import DATA_DIR, BAD_FILE_FORMAT, extract_metadata, CPY_VERSION, _get_modules_file + +#: The location to store a local copy of code.py for use with --auto and +# web workflow +LOCAL_CODE_PY_COPY = os.path.join(DATA_DIR, "code.tmp.py") + +class Backend: + """ + Backend parent class to be extended for workflow specific + implementations + """ + + def __init__(self, logger): + self.LIB_DIR_PATH = None + self.logger = logger + + def get_circuitpython_version(self, device_location): + """ + Must be overridden by subclass for implementation! + + Returns the version number of CircuitPython running on the board connected + via ``device_url``, along with the board ID. + + :param str device_location: http based device URL or local file path. + :return: A tuple with the version string for CircuitPython and the board ID string. + """ + raise NotImplementedError + + def _get_modules(self, device_lib_path): + """ + To be overridden by subclass + """ + raise NotImplementedError + + def get_modules(self, device_url): + """ + Get a dictionary containing metadata about all the Python modules found in + the referenced path. + + :param str device_url: URL to be used to find modules. + :return: A dictionary containing metadata about the found modules. + """ + return self._get_modules(device_url) + + def get_device_versions(self, device_url): + """ + Returns a dictionary of metadata from modules on the connected device. + + :param str device_url: URL for the device. + :return: A dictionary of metadata about the modules available on the + connected device. + """ + return self.get_modules(os.path.join(device_url, self.LIB_DIR_PATH)) + + def _create_library_directory(self, device_path, library_path): + """ + To be overridden by subclass + """ + raise NotImplementedError + + def _install_module_py(self, library_path, metadata): + """ + To be overridden by subclass + """ + raise NotImplementedError + + def _install_module_mpy(self, bundle, library_path, metadata): + """ + To be overridden by subclass + """ + raise NotImplementedError + + # pylint: disable=too-many-locals,too-many-branches,too-many-arguments + def install_module( + self, device_path, device_modules, name, pyext, mod_names + ): # pragma: no cover + """ + Finds a connected device and installs a given module name if it + is available in the current module bundle and is not already + installed on the device. + TODO: There is currently no check for the version. + + :param str device_path: The path to the connected board. + :param list(dict) device_modules: List of module metadata from the device. + :param str name: Name of module to install + :param bool pyext: Boolean to specify if the module should be installed from + source or from a pre-compiled module + :param mod_names: Dictionary of metadata from modules that can be generated + with get_bundle_versions() + """ + if not name: + click.echo("No module name(s) provided.") + elif name in mod_names: + # Grab device modules to check if module already installed + if name in device_modules: + click.echo("'{}' is already installed.".format(name)) + return + + library_path = os.path.join(device_path, self.LIB_DIR_PATH) + + # Create the library directory first. + self._create_library_directory(device_path, library_path) + + metadata = mod_names[name] + bundle = metadata["bundle"] + if pyext: + # Use Python source for module. + self._install_module_py(library_path, metadata) + else: + # Use pre-compiled mpy modules. + self._install_module_mpy(bundle, library_path, metadata) + click.echo("Installed '{}'.".format(name)) + else: + click.echo("Unknown module named, '{}'.".format(name)) + + def libraries_from_imports(self, code_py, mod_names): + """ + To be overridden by subclass + """ + raise NotImplementedError + + def uninstall(self, device_path, module_path): + """ + To be overridden by subclass + """ + raise NotImplementedError + + def update(self, module): + """ + To be overridden by subclass + """ + raise NotImplementedError + + def libraries_from_code_py(self, code_py, mod_names): + """ + Parse the given code.py file and return the imported libraries + + :param str code_py: Full path of the code.py file + :return: sequence of library names + """ + # pylint: disable=broad-except + try: + found_imports = findimports.find_imports(code_py) + except Exception as ex: # broad exception because anything could go wrong + self.logger.exception(ex) + click.secho('Unable to read the auto file: "{}"'.format(str(ex)), fg="red") + sys.exit(2) + # pylint: enable=broad-except + imports = [info.name.split(".", 1)[0] for info in found_imports] + return [r for r in imports if r in mod_names] + +class WebBackend(Backend): + """ + Backend for interacting with a device via Web Workflow + """ + + def __init__(self, host, password, logger): + super().__init__(logger) + self.LIB_DIR_PATH = "fs/lib/" + self.host = host + self.password = password + + def install_file_http(self, source, target): + """ + Install file to device using web workflow. + :param source source file. + :param target destination URL. Should have password embedded. + """ + url = urlparse(target) + auth = HTTPBasicAuth("", url.password) + print(f"target: {target}") + + with open(source, "rb") as fp: + r = requests.put(target, fp.read(), auth=auth) + r.raise_for_status() + + def install_dir_http(self, source, target): + """ + Install directory to device using web workflow. + :param source source directory. + :param target destination URL. Should have password embedded. + """ + url = urlparse(target) + auth = HTTPBasicAuth("", url.password) + print(f"target: {target}") + + # Create the top level directory. + r = requests.put(target + ("/" if not target.endswith("/") else ""), auth=auth) + r.raise_for_status() + + # Traverse the directory structure and create the directories/files. + for root, dirs, files in os.walk(source): + rel_path = os.path.relpath(root, source) + if rel_path == ".": + rel_path = "" + for name in files: + with open(os.path.join(root, name), "rb") as fp: + r = requests.put( + target + rel_path + "/" + name, fp.read(), auth=auth + ) + r.raise_for_status() + for name in dirs: + r = requests.put(target + rel_path + "/" + name, auth=auth) + r.raise_for_status() + + def get_circuitpython_version(self, url): + """ + Returns the version number of CircuitPython running on the board connected + via ``device_path``, along with the board ID. This is obtained using + RESTful API from the /cp/version.json URL. + + :param str url: board URL. + :return: A tuple with the version string for CircuitPython and the board ID string. + """ + # pylint: disable=arguments-renamed + r = requests.get(url + "/cp/version.json") + # pylint: disable=no-member + if r.status_code != requests.codes.ok: + click.secho( + f" Unable to get version from {url}: {r.status_code}", fg="red" + ) + sys.exit(1) + # pylint: enable=no-member + ver_json = r.json() + return ver_json.get("version"), ver_json.get("board_id") + + def _get_modules(self, device_lib_path): + return self._get_modules_http(device_lib_path) + + def _get_modules_http(self, url): + """ + Get a dictionary containing metadata about all the Python modules found using + the referenced URL. + + :param str url: URL for the modules. + :return: A dictionary containing metadata about the found modules. + """ + result = {} + u = urlparse(url) + auth = HTTPBasicAuth("", u.password) + r = requests.get(url, auth=auth, headers={"Accept": "application/json"}) + r.raise_for_status() + + directory_mods = [] + single_file_mods = [] + for entry in r.json(): + entry_name = entry.get("name") + if entry.get("directory"): + directory_mods.append(entry_name) + else: + if entry_name.endswith(".py") or entry_name.endswith(".mpy"): + single_file_mods.append(entry_name) + + self._get_modules_http_single_mods(auth, result, single_file_mods, url) + self._get_modules_http_dir_mods(auth, directory_mods, result, url) + + return result + + def _get_modules_http_dir_mods(self, auth, directory_mods, result, url): + """ + #TODO describe what this does + + :param auth HTTP authentication. + :param directory_mods list of modules. + :param result dictionary for the result. + :param url: URL of the device. + """ + for dm in directory_mods: + dm_url = url + dm + "/" + r = requests.get(dm_url, auth=auth, headers={"Accept": "application/json"}) + r.raise_for_status() + mpy = False + for entry in r.json(): + entry_name = entry.get("name") + if not entry.get("directory") and ( + entry_name.endswith(".py") or entry_name.endswith(".mpy") + ): + if entry_name.endswith(".mpy"): + mpy = True + r = requests.get(dm_url + entry_name, auth=auth) + r.raise_for_status() + idx = entry_name.rfind(".") + with tempfile.NamedTemporaryFile( + prefix=entry_name[:idx] + "-", + suffix=entry_name[idx:], + delete=False, + ) as fp: + fp.write(r.content) + tmp_name = fp.name + metadata = extract_metadata(tmp_name, self.logger) + os.remove(tmp_name) + if "__version__" in metadata: + metadata["path"] = dm_url + result[dm] = metadata + # break now if any of the submodules has a bad format + if metadata["__version__"] == BAD_FILE_FORMAT: + break + + if result.get(dm) is None: + result[dm] = {"path": dm_url, "mpy": mpy} + + def _get_modules_http_single_mods(self, auth, result, single_file_mods, url): + """ + :param auth HTTP authentication. + :param single_file_mods list of modules. + :param result dictionary for the result. + :param url: URL of the device. + """ + for sfm in single_file_mods: + sfm_url = url + sfm + r = requests.get(sfm_url, auth=auth) + r.raise_for_status() + idx = sfm.rfind(".") + with tempfile.NamedTemporaryFile( + prefix=sfm[:idx] + "-", suffix=sfm[idx:], delete=False + ) as fp: + fp.write(r.content) + tmp_name = fp.name + metadata = extract_metadata(tmp_name, self.logger) + os.remove(tmp_name) + metadata["path"] = sfm_url + result[sfm[:idx]] = metadata + + def _create_library_directory(self, device_path, library_path): + url = urlparse(device_path) + auth = HTTPBasicAuth("", url.password) + r = requests.put(library_path, auth=auth) + r.raise_for_status() + + def _install_module_mpy(self, bundle, library_path, metadata): + """ + :param bundle library bundle. + :param library_path library path + :param metadata dictionary. + """ + module_name = os.path.basename(metadata["path"]).replace(".py", ".mpy") + if not module_name: + # Must be a directory based module. + module_name = os.path.basename(os.path.dirname(metadata["path"])) + major_version = CPY_VERSION.split(".")[0] + bundle_platform = "{}mpy".format(major_version) + bundle_path = os.path.join(bundle.lib_dir(bundle_platform), module_name) + if os.path.isdir(bundle_path): + + self.install_dir_http(bundle_path, library_path + module_name) + + elif os.path.isfile(bundle_path): + target = os.path.basename(bundle_path) + + self.install_file_http(bundle_path, library_path + target) + + else: + raise IOError("Cannot find compiled version of module.") + + # pylint: enable=too-many-locals,too-many-branches + def _install_module_py(self, library_path, metadata): + """ + :param library_path library path + :param metadata dictionary. + """ + source_path = metadata["path"] # Path to Python source version. + if os.path.isdir(source_path): + target = os.path.basename(os.path.dirname(source_path)) + self.install_dir_http(source_path, library_path + target) + + else: + target = os.path.basename(source_path) + self.install_file_http(source_path, library_path + target) + + def libraries_from_imports(self, code_py, mod_names): + """ + Parse the given code.py file and return the imported libraries + + :param str code_py: Full path of the code.py file + :return: sequence of library names + """ + url = code_py + auth = HTTPBasicAuth("", self.password) + r = requests.get(url, auth=auth) + r.raise_for_status() + with open(LOCAL_CODE_PY_COPY, "w", encoding="utf-8") as f: + f.write(r.text) + code_py = LOCAL_CODE_PY_COPY + + return self.libraries_from_code_py(code_py, mod_names) + + def uninstall(self, device_path, module_path): + """ + Uninstall given module on device using REST API. + """ + url = urlparse(device_path) + auth = HTTPBasicAuth("", url.password) + r = requests.delete(module_path, auth=auth) + r.raise_for_status() + + def update(self, module): + """ + Delete the module on the device, then copy the module from the bundle + back onto the device. + + The caller is expected to handle any exceptions raised. + """ + self._update_http(module) + + def _update_http(self, module): + """ + Update the module using web workflow. + """ + if module.file: + # Copy the file (will overwrite). + self.install_file_http(module.bundle_path, module.path) + else: + # Delete the directory (recursive) first. + url = urlparse(module.path) + auth = HTTPBasicAuth("", url.password) + r = requests.delete(module.path, auth=auth) + r.raise_for_status() + self.install_dir_http(module.bundle_path, module.path) + +class DiskBackend(Backend): + """ + Backend for interacting with a device via USB Workflow + """ + + def __init__(self, device_location, logger): + super().__init__(logger) + self.LIB_DIR_PATH = "lib" + self.device_location = device_location + + def get_circuitpython_version(self, device_location): + """ + Returns the version number of CircuitPython running on the board connected + via ``device_path``, along with the board ID. This is obtained from the + ``boot_out.txt`` file on the device, whose first line will start with + something like this:: + + Adafruit CircuitPython 4.1.0 on 2019-08-02; + + While the second line is:: + + Board ID:raspberry_pi_pico + + :param str device_path: The path to the connected board. + :return: A tuple with the version string for CircuitPython and the board ID string. + """ + try: + with open( + os.path.join(device_location, "boot_out.txt"), "r", encoding="utf-8" + ) as boot: + version_line = boot.readline() + circuit_python = version_line.split(";")[0].split(" ")[-3] + board_line = boot.readline() + if board_line.startswith("Board ID:"): + board_id = board_line[9:].strip() + else: + board_id = "" + except FileNotFoundError: + click.secho( + "Missing file boot_out.txt on the device: wrong path or drive corrupted.", + fg="red", + ) + self.logger.error("boot_out.txt not found.") + sys.exit(1) + + return circuit_python, board_id + + def _get_modules(self, device_lib_path): + """ + Get a dictionary containing metadata about all the Python modules found in + the referenced path. + + :param str device_lib_path: URL to be used to find modules. + :return: A dictionary containing metadata about the found modules. + """ + return _get_modules_file(device_lib_path, self.logger) + + def _create_library_directory(self, device_path, library_path): + if not os.path.exists(library_path): # pragma: no cover + os.makedirs(library_path) + + def _install_module_mpy(self, bundle, library_path, metadata): + """ + :param bundle library bundle. + :param library_path library path + :param metadata dictionary. + """ + module_name = os.path.basename(metadata["path"]).replace(".py", ".mpy") + if not module_name: + # Must be a directory based module. + module_name = os.path.basename(os.path.dirname(metadata["path"])) + + major_version = self.get_circuitpython_version(self.device_location)[0].split(".")[0] + bundle_platform = "{}mpy".format(major_version) + bundle_path = os.path.join(bundle.lib_dir(bundle_platform), module_name) + if os.path.isdir(bundle_path): + target_path = os.path.join(library_path, module_name) + # Copy the directory. + shutil.copytree(bundle_path, target_path) + elif os.path.isfile(bundle_path): + target = os.path.basename(bundle_path) + target_path = os.path.join(library_path, target) + # Copy file. + shutil.copyfile(bundle_path, target_path) + else: + raise IOError("Cannot find compiled version of module.") + + # pylint: enable=too-many-locals,too-many-branches + def _install_module_py(self, library_path, metadata): + """ + :param library_path library path + :param metadata dictionary. + """ + + source_path = metadata["path"] # Path to Python source version. + if os.path.isdir(source_path): + target = os.path.basename(os.path.dirname(source_path)) + target_path = os.path.join(library_path, target) + # Copy the directory. + shutil.copytree(source_path, target_path) + else: + target = os.path.basename(source_path) + target_path = os.path.join(library_path, target) + # Copy file. + shutil.copyfile(source_path, target_path) + + def libraries_from_imports(self, code_py, mod_names): + """ + Parse the given code.py file and return the imported libraries + + :param str code_py: Full path of the code.py file + :return: sequence of library names + """ + return self.libraries_from_code_py(code_py, mod_names) + + def uninstall(self, device_path, module_path): + """ + Uninstall module using local file system. + """ + library_path = os.path.join(device_path, "lib") + if os.path.isdir(module_path): + target = os.path.basename(os.path.dirname(module_path)) + target_path = os.path.join(library_path, target) + # Remove the directory. + shutil.rmtree(target_path) + else: + target = os.path.basename(module_path) + target_path = os.path.join(library_path, target) + # Remove file + os.remove(target_path) + + def update(self, module): + """ + Delete the module on the device, then copy the module from the bundle + back onto the device. + + The caller is expected to handle any exceptions raised. + """ + self._update_file(module) + + def _update_file(self, module): + """ + Update the module using file system. + """ + if os.path.isdir(module.path): + # Delete and copy the directory. + shutil.rmtree(module.path, ignore_errors=True) + shutil.copytree(module.bundle_path, module.path) + else: + # Delete and copy file. + os.remove(module.path) + shutil.copyfile(module.bundle_path, module.path) \ No newline at end of file diff --git a/circup/shared.py b/circup/shared.py new file mode 100644 index 0000000..cc60166 --- /dev/null +++ b/circup/shared.py @@ -0,0 +1,127 @@ +import glob +import os +import re + +import appdirs + +#: Version identifier for a bad MPY file format +BAD_FILE_FORMAT = "Invalid" + +#: The location of data files used by circup (following OS conventions). +DATA_DIR = appdirs.user_data_dir(appname="circup", appauthor="adafruit") + +#: The version of CircuitPython found on the connected device. +CPY_VERSION = "" + + +def _get_modules_file(path, logger): + """ + Get a dictionary containing metadata about all the Python modules found in + the referenced file system path. + + :param str path: The directory in which to find modules. + :return: A dictionary containing metadata about the found modules. + """ + result = {} + if not path: + return result + single_file_py_mods = glob.glob(os.path.join(path, "*.py")) + single_file_mpy_mods = glob.glob(os.path.join(path, "*.mpy")) + package_dir_mods = [ + d + for d in glob.glob(os.path.join(path, "*", "")) + if not os.path.basename(os.path.normpath(d)).startswith(".") + ] + single_file_mods = single_file_py_mods + single_file_mpy_mods + for sfm in [f for f in single_file_mods if not os.path.basename(f).startswith(".")]: + metadata = extract_metadata(sfm, logger) + metadata["path"] = sfm + result[os.path.basename(sfm).replace(".py", "").replace(".mpy", "")] = metadata + for package_path in package_dir_mods: + name = os.path.basename(os.path.dirname(package_path)) + py_files = glob.glob(os.path.join(package_path, "**/*.py"), recursive=True) + mpy_files = glob.glob(os.path.join(package_path, "**/*.mpy"), recursive=True) + all_files = py_files + mpy_files + # default value + result[name] = {"path": package_path, "mpy": bool(mpy_files)} + # explore all the submodules to detect bad ones + for source in [f for f in all_files if not os.path.basename(f).startswith(".")]: + metadata = extract_metadata(source, logger) + if "__version__" in metadata: + metadata["path"] = package_path + result[name] = metadata + # break now if any of the submodules has a bad format + if metadata["__version__"] == BAD_FILE_FORMAT: + break + return result + +def extract_metadata(path, logger): + """ + Given a file path, return a dictionary containing metadata extracted from + dunder attributes found therein. Works with both .py and .mpy files. + + For Python source files, such metadata assignments should be simple and + single-line. For example:: + + __version__ = "1.1.4" + __repo__ = "https://github.com/adafruit/SomeLibrary.git" + + For byte compiled .mpy files, a brute force / backtrack approach is used + to find the __version__ number in the file -- see comments in the + code for the implementation details. + + :param str path: The path to the file containing the metadata. + :return: The dunder based metadata found in the file, as a dictionary. + """ + result = {} + logger.info("%s", path) + if path.endswith(".py"): + result["mpy"] = False + with open(path, "r", encoding="utf-8") as source_file: + content = source_file.read() + #: The regex used to extract ``__version__`` and ``__repo__`` assignments. + dunder_key_val = r"""(__\w+__)(?:\s*:\s*\w+)?\s*=\s*(?:['"]|\(\s)(.+)['"]""" + for match in re.findall(dunder_key_val, content): + result[match[0]] = str(match[1]) + elif path.endswith(".mpy"): + result["mpy"] = True + with open(path, "rb") as mpy_file: + content = mpy_file.read() + # Track the MPY version number + mpy_version = content[0:2] + compatibility = None + loc = -1 + # Find the start location of the __version__ + if mpy_version == b"M\x03": + # One byte for the length of "__version__" + loc = content.find(b"__version__") - 1 + compatibility = (None, "7.0.0-alpha.1") + elif mpy_version == b"C\x05": + # Two bytes in mpy version 5 + loc = content.find(b"__version__") - 2 + compatibility = ("7.0.0-alpha.1", None) + if loc > -1: + # Backtrack until a byte value of the offset is reached. + offset = 1 + while offset < loc: + val = int(content[loc - offset]) + if mpy_version == b"C\x05": + val = val // 2 + if val == offset - 1: # Off by one..! + # Found version, extract the number given boundaries. + start = loc - offset + 1 # No need for prepended length. + end = loc # Up to the start of the __version__. + version = content[start:end] # Slice the version number. + # Create a string version as metadata in the result. + result["__version__"] = version.decode("utf-8") + break # Nothing more to do. + offset += 1 # ...and again but backtrack by one. + if compatibility: + result["compatibility"] = compatibility + else: + # not a valid MPY file + result["__version__"] = BAD_FILE_FORMAT + + if result: + logger.info("Extracted metadata: %s", result) + return result \ No newline at end of file From 499dee6e03a025485d18cc8d0ec48cc4e57192ee Mon Sep 17 00:00:00 2001 From: foamyguy Date: Thu, 30 Nov 2023 06:44:22 -0600 Subject: [PATCH 24/63] move libraries_from_code_py out of backend. fix CPY_VERSION issue for web backend --- circup/__init__.py | 42 +++++++++++++++++++++++++++--------------- circup/backends.py | 45 ++++++++++++++++++++++++--------------------- circup/shared.py | 6 ++---- 3 files changed, 53 insertions(+), 40 deletions(-) diff --git a/circup/__init__.py b/circup/__init__.py index 2516dba..58dd282 100644 --- a/circup/__init__.py +++ b/circup/__init__.py @@ -30,9 +30,12 @@ from requests.auth import HTTPBasicAuth from semver import VersionInfo -from circup.shared import DATA_DIR, BAD_FILE_FORMAT, extract_metadata, CPY_VERSION, _get_modules_file +from circup.shared import DATA_DIR, BAD_FILE_FORMAT, extract_metadata, _get_modules_file from circup.backends import WebBackend, DiskBackend +#: The version of CircuitPython found on the connected device. +CPY_VERSION = "" + # Useful constants. #: Flag to indicate if the command is being run in verbose mode. VERBOSE = False @@ -410,13 +413,6 @@ def __repr__(self): ) - - - - - - - def clean_library_name(assumed_library_name): """ Most CP repos and library names are look like this: @@ -505,9 +501,6 @@ def ensure_latest_bundle(bundle): logger.info("Current bundle up to date %s.", tag) - - - def find_device(): """ Return the location on the filesystem for the connected CircuitPython device. @@ -936,6 +929,25 @@ def tags_data_save_tag(key, tag): json.dump(tags_data, data) +def libraries_from_code_py(code_py, mod_names): + """ + Parse the given code.py file and return the imported libraries + + :param str code_py: Full path of the code.py file + :return: sequence of library names + """ + # pylint: disable=broad-except + try: + found_imports = findimports.find_imports(code_py) + except Exception as ex: # broad exception because anything could go wrong + self.logger.exception(ex) + click.secho('Unable to read the auto file: "{}"'.format(str(ex)), fg="red") + sys.exit(2) + # pylint: enable=broad-except + imports = [info.name.split(".", 1)[0] for info in found_imports] + return [r for r in imports if r in mod_names] + + # ----------- CLI command definitions ----------- # # The following functions have IO side effects (for instance they emit to @@ -1014,7 +1026,6 @@ def main(ctx, verbose, path, host, password, board_id, cpy_version): # pragma: if ctx.invoked_subcommand in BOARDLESS_COMMANDS: return - ctx.obj["DEVICE_PATH"] = device_path latest_version = get_latest_release_from_url( "https://github.com/adafruit/circuitpython/releases/latest" @@ -1213,9 +1224,10 @@ def install(ctx, modules, pyext, requirement, auto, auto_file): # pragma: no co if not os.path.isfile(auto_file) and not ctx.obj["using_webworkflow"]: click.secho(f"Auto file not found: {auto_file}", fg="red") sys.exit(1) - requested_installs = ctx.obj["backend"].libraries_from_imports( - auto_file, mod_names - ) + + auto_file_path = ctx.obj["backend"].get_auto_file_path(auto_file) + + requested_installs = libraries_from_code_py(auto_file_path, mod_names) else: requested_installs = modules requested_installs = sorted(set(requested_installs)) diff --git a/circup/backends.py b/circup/backends.py index 9edfc5e..96d9d90 100644 --- a/circup/backends.py +++ b/circup/backends.py @@ -9,12 +9,13 @@ import requests from requests.auth import HTTPBasicAuth -from circup.shared import DATA_DIR, BAD_FILE_FORMAT, extract_metadata, CPY_VERSION, _get_modules_file +from circup.shared import DATA_DIR, BAD_FILE_FORMAT, extract_metadata, _get_modules_file #: The location to store a local copy of code.py for use with --auto and # web workflow LOCAL_CODE_PY_COPY = os.path.join(DATA_DIR, "code.tmp.py") + class Backend: """ Backend parent class to be extended for workflow specific @@ -142,23 +143,6 @@ def update(self, module): """ raise NotImplementedError - def libraries_from_code_py(self, code_py, mod_names): - """ - Parse the given code.py file and return the imported libraries - - :param str code_py: Full path of the code.py file - :return: sequence of library names - """ - # pylint: disable=broad-except - try: - found_imports = findimports.find_imports(code_py) - except Exception as ex: # broad exception because anything could go wrong - self.logger.exception(ex) - click.secho('Unable to read the auto file: "{}"'.format(str(ex)), fg="red") - sys.exit(2) - # pylint: enable=broad-except - imports = [info.name.split(".", 1)[0] for info in found_imports] - return [r for r in imports if r in mod_names] class WebBackend(Backend): """ @@ -170,6 +154,7 @@ def __init__(self, host, password, logger): self.LIB_DIR_PATH = "fs/lib/" self.host = host self.password = password + self.device_location = f"http://:{self.password}@{self.host}" def install_file_http(self, source, target): """ @@ -348,7 +333,9 @@ def _install_module_mpy(self, bundle, library_path, metadata): if not module_name: # Must be a directory based module. module_name = os.path.basename(os.path.dirname(metadata["path"])) - major_version = CPY_VERSION.split(".")[0] + major_version = self.get_circuitpython_version(self.device_location)[0].split( + "." + )[0] bundle_platform = "{}mpy".format(major_version) bundle_path = os.path.join(bundle.lib_dir(bundle_platform), module_name) if os.path.isdir(bundle_path): @@ -378,6 +365,16 @@ def _install_module_py(self, library_path, metadata): target = os.path.basename(source_path) self.install_file_http(source_path, library_path + target) + def get_auto_file_path(self, auto_file_path): + url = auto_file_path + auth = HTTPBasicAuth("", self.password) + r = requests.get(url, auth=auth) + r.raise_for_status() + with open(LOCAL_CODE_PY_COPY, "w", encoding="utf-8") as f: + f.write(r.text) + LOCAL_CODE_PY_COPY + return LOCAL_CODE_PY_COPY + def libraries_from_imports(self, code_py, mod_names): """ Parse the given code.py file and return the imported libraries @@ -428,6 +425,7 @@ def _update_http(self, module): r.raise_for_status() self.install_dir_http(module.bundle_path, module.path) + class DiskBackend(Backend): """ Backend for interacting with a device via USB Workflow @@ -500,7 +498,9 @@ def _install_module_mpy(self, bundle, library_path, metadata): # Must be a directory based module. module_name = os.path.basename(os.path.dirname(metadata["path"])) - major_version = self.get_circuitpython_version(self.device_location)[0].split(".")[0] + major_version = self.get_circuitpython_version(self.device_location)[0].split( + "." + )[0] bundle_platform = "{}mpy".format(major_version) bundle_path = os.path.join(bundle.lib_dir(bundle_platform), module_name) if os.path.isdir(bundle_path): @@ -534,6 +534,9 @@ def _install_module_py(self, library_path, metadata): # Copy file. shutil.copyfile(source_path, target_path) + def get_auto_file_path(self, auto_file_path): + return auto_file_path + def libraries_from_imports(self, code_py, mod_names): """ Parse the given code.py file and return the imported libraries @@ -579,4 +582,4 @@ def _update_file(self, module): else: # Delete and copy file. os.remove(module.path) - shutil.copyfile(module.bundle_path, module.path) \ No newline at end of file + shutil.copyfile(module.bundle_path, module.path) diff --git a/circup/shared.py b/circup/shared.py index cc60166..5f157b1 100644 --- a/circup/shared.py +++ b/circup/shared.py @@ -10,9 +10,6 @@ #: The location of data files used by circup (following OS conventions). DATA_DIR = appdirs.user_data_dir(appname="circup", appauthor="adafruit") -#: The version of CircuitPython found on the connected device. -CPY_VERSION = "" - def _get_modules_file(path, logger): """ @@ -55,6 +52,7 @@ def _get_modules_file(path, logger): break return result + def extract_metadata(path, logger): """ Given a file path, return a dictionary containing metadata extracted from @@ -124,4 +122,4 @@ def extract_metadata(path, logger): if result: logger.info("Extracted metadata: %s", result) - return result \ No newline at end of file + return result From bea088ec506fea951ec4fec55920e7806c40e478 Mon Sep 17 00:00:00 2001 From: foamyguy Date: Sat, 2 Dec 2023 11:57:24 -0600 Subject: [PATCH 25/63] don't pass library path. --- circup/__init__.py | 2 -- circup/backends.py | 67 ++++++++++++++++++---------------------------- 2 files changed, 26 insertions(+), 43 deletions(-) diff --git a/circup/__init__.py b/circup/__init__.py index 58dd282..0d1ca9d 100644 --- a/circup/__init__.py +++ b/circup/__init__.py @@ -130,8 +130,6 @@ def lib_dir(self, platform): :return: The path to the lib directory for the platform. """ tag = self.current_tag - print(platform) - print(PLATFORMS) return os.path.join( self.dir.format(platform=platform), self.basename.format(platform=PLATFORMS[platform], tag=tag), diff --git a/circup/backends.py b/circup/backends.py index 96d9d90..3f6ff19 100644 --- a/circup/backends.py +++ b/circup/backends.py @@ -70,13 +70,13 @@ def _create_library_directory(self, device_path, library_path): """ raise NotImplementedError - def _install_module_py(self, library_path, metadata): + def _install_module_py(self, metadata): """ To be overridden by subclass """ raise NotImplementedError - def _install_module_mpy(self, bundle, library_path, metadata): + def _install_module_mpy(self, bundle, metadata): """ To be overridden by subclass """ @@ -117,10 +117,10 @@ def install_module( bundle = metadata["bundle"] if pyext: # Use Python source for module. - self._install_module_py(library_path, metadata) + self._install_module_py(metadata) else: # Use pre-compiled mpy modules. - self._install_module_mpy(bundle, library_path, metadata) + self._install_module_mpy(bundle, metadata) click.echo("Installed '{}'.".format(name)) else: click.echo("Unknown module named, '{}'.".format(name)) @@ -156,6 +156,8 @@ def __init__(self, host, password, logger): self.password = password self.device_location = f"http://:{self.password}@{self.host}" + self.library_path = self.device_location + "/" + self.LIB_DIR_PATH + def install_file_http(self, source, target): """ Install file to device using web workflow. @@ -182,6 +184,7 @@ def install_dir_http(self, source, target): # Create the top level directory. r = requests.put(target + ("/" if not target.endswith("/") else ""), auth=auth) + print(f"resp {r.content}") r.raise_for_status() # Traverse the directory structure and create the directories/files. @@ -323,12 +326,14 @@ def _create_library_directory(self, device_path, library_path): r = requests.put(library_path, auth=auth) r.raise_for_status() - def _install_module_mpy(self, bundle, library_path, metadata): + def _install_module_mpy(self, bundle, metadata): """ :param bundle library bundle. :param library_path library path :param metadata dictionary. """ + library_path = self.library_path + print(f"metadata: {metadata}") module_name = os.path.basename(metadata["path"]).replace(".py", ".mpy") if not module_name: # Must be a directory based module. @@ -340,30 +345,35 @@ def _install_module_mpy(self, bundle, library_path, metadata): bundle_path = os.path.join(bundle.lib_dir(bundle_platform), module_name) if os.path.isdir(bundle_path): + print(f"456 library_path: {library_path}") + print(f"456 module_name: {module_name}") + self.install_dir_http(bundle_path, library_path + module_name) elif os.path.isfile(bundle_path): target = os.path.basename(bundle_path) - + print(f"123 library_path: {library_path}") + print(f"123 target: {target}") self.install_file_http(bundle_path, library_path + target) else: raise IOError("Cannot find compiled version of module.") # pylint: enable=too-many-locals,too-many-branches - def _install_module_py(self, library_path, metadata): + def _install_module_py(self, metadata): """ :param library_path library path :param metadata dictionary. """ + source_path = metadata["path"] # Path to Python source version. if os.path.isdir(source_path): target = os.path.basename(os.path.dirname(source_path)) - self.install_dir_http(source_path, library_path + target) + self.install_dir_http(source_path, self.library_path + target) else: target = os.path.basename(source_path) - self.install_file_http(source_path, library_path + target) + self.install_file_http(source_path, self.library_path + target) def get_auto_file_path(self, auto_file_path): url = auto_file_path @@ -375,23 +385,6 @@ def get_auto_file_path(self, auto_file_path): LOCAL_CODE_PY_COPY return LOCAL_CODE_PY_COPY - def libraries_from_imports(self, code_py, mod_names): - """ - Parse the given code.py file and return the imported libraries - - :param str code_py: Full path of the code.py file - :return: sequence of library names - """ - url = code_py - auth = HTTPBasicAuth("", self.password) - r = requests.get(url, auth=auth) - r.raise_for_status() - with open(LOCAL_CODE_PY_COPY, "w", encoding="utf-8") as f: - f.write(r.text) - code_py = LOCAL_CODE_PY_COPY - - return self.libraries_from_code_py(code_py, mod_names) - def uninstall(self, device_path, module_path): """ Uninstall given module on device using REST API. @@ -435,6 +428,7 @@ def __init__(self, device_location, logger): super().__init__(logger) self.LIB_DIR_PATH = "lib" self.device_location = device_location + self.library_path = self.device_location + "/" + self.LIB_DIR_PATH def get_circuitpython_version(self, device_location): """ @@ -487,7 +481,7 @@ def _create_library_directory(self, device_path, library_path): if not os.path.exists(library_path): # pragma: no cover os.makedirs(library_path) - def _install_module_mpy(self, bundle, library_path, metadata): + def _install_module_mpy(self, bundle, metadata): """ :param bundle library bundle. :param library_path library path @@ -504,19 +498,19 @@ def _install_module_mpy(self, bundle, library_path, metadata): bundle_platform = "{}mpy".format(major_version) bundle_path = os.path.join(bundle.lib_dir(bundle_platform), module_name) if os.path.isdir(bundle_path): - target_path = os.path.join(library_path, module_name) + target_path = os.path.join(self.library_path, module_name) # Copy the directory. shutil.copytree(bundle_path, target_path) elif os.path.isfile(bundle_path): target = os.path.basename(bundle_path) - target_path = os.path.join(library_path, target) + target_path = os.path.join(self.library_path, target) # Copy file. shutil.copyfile(bundle_path, target_path) else: raise IOError("Cannot find compiled version of module.") # pylint: enable=too-many-locals,too-many-branches - def _install_module_py(self, library_path, metadata): + def _install_module_py(self, metadata): """ :param library_path library path :param metadata dictionary. @@ -525,27 +519,18 @@ def _install_module_py(self, library_path, metadata): source_path = metadata["path"] # Path to Python source version. if os.path.isdir(source_path): target = os.path.basename(os.path.dirname(source_path)) - target_path = os.path.join(library_path, target) + target_path = os.path.join(self.library_path, target) # Copy the directory. shutil.copytree(source_path, target_path) else: target = os.path.basename(source_path) - target_path = os.path.join(library_path, target) + target_path = os.path.join(self.library_path, target) # Copy file. shutil.copyfile(source_path, target_path) def get_auto_file_path(self, auto_file_path): return auto_file_path - def libraries_from_imports(self, code_py, mod_names): - """ - Parse the given code.py file and return the imported libraries - - :param str code_py: Full path of the code.py file - :return: sequence of library names - """ - return self.libraries_from_code_py(code_py, mod_names) - def uninstall(self, device_path, module_path): """ Uninstall module using local file system. From d9ce3b25a95ebc3abf8aa0d675ffd1da636d0d05 Mon Sep 17 00:00:00 2001 From: foamyguy Date: Mon, 4 Dec 2023 15:19:00 -0600 Subject: [PATCH 26/63] copyright for shared.py and backends.py --- circup/backends.py | 7 +++++++ circup/shared.py | 8 ++++++++ 2 files changed, 15 insertions(+) diff --git a/circup/backends.py b/circup/backends.py index 3f6ff19..bda7afc 100644 --- a/circup/backends.py +++ b/circup/backends.py @@ -1,3 +1,10 @@ +# SPDX-FileCopyrightText: 2019 Nicholas Tollervey, written for Adafruit Industries +# SPDX-FileCopyrightText: 2023 Tim Cocks, written for Adafruit Industries +# +# SPDX-License-Identifier: MIT +""" +Backend classes that represent interfaces to physical devices. +""" import os import shutil import sys diff --git a/circup/shared.py b/circup/shared.py index 5f157b1..786c0ca 100644 --- a/circup/shared.py +++ b/circup/shared.py @@ -1,3 +1,11 @@ +# SPDX-FileCopyrightText: 2019 Nicholas Tollervey, written for Adafruit Industries +# SPDX-FileCopyrightText: 2023 Tim Cocks, written for Adafruit Industries +# +# SPDX-License-Identifier: MIT +""" +Utilities that are shared and used by both click CLI command functions +and Backend class functions. +""" import glob import os import re From 01f0a2d046f0ab6f34098efc52ff37cab5d913f9 Mon Sep 17 00:00:00 2001 From: foamyguy Date: Mon, 4 Dec 2023 16:50:29 -0600 Subject: [PATCH 27/63] removing path argument from backend functions and find_modules(). add backend.get_file_path(). remove target url argument (including passwrd) from install_http() --- circup/__init__.py | 41 ++++++++--------------- circup/backends.py | 82 +++++++++++++++++++++++++++++++--------------- 2 files changed, 69 insertions(+), 54 deletions(-) diff --git a/circup/__init__.py b/circup/__init__.py index 0d1ca9d..b8303f8 100644 --- a/circup/__init__.py +++ b/circup/__init__.py @@ -567,20 +567,20 @@ def get_volume_name(disk_name): return device_dir -def find_modules(backend, device_url, bundles_list): +def find_modules(backend, bundles_list): """ Extracts metadata from the connected device and available bundles and returns this as a list of Module instances representing the modules on the device. - :param str device_url: The URL to the board. + :param Backend backend: Backend with the device connection. :param List[Bundle] bundles_list: List of supported bundles as Bundle objects. :return: A list of Module instances describing the current state of the modules on the connected device. """ # pylint: disable=broad-except,too-many-locals try: - device_modules = backend.get_device_versions(device_url) + device_modules = backend.get_device_versions() bundle_modules = get_bundle_versions(bundles_list) result = [] for name, device_metadata in device_modules.items(): @@ -997,7 +997,6 @@ def main(ctx, verbose, path, host, password, board_id, cpy_version): # pragma: device_path = get_device_path(host, password, path) using_webworkflow = "host" in ctx.params.keys() and ctx.params["host"] is not None - ctx.obj["using_webworkflow"] = using_webworkflow if using_webworkflow: ctx.obj["backend"] = WebBackend(host=host, password=password, logger=logger) else: @@ -1034,7 +1033,7 @@ def main(ctx, verbose, path, host, password, board_id, cpy_version): # pragma: sys.exit(1) else: CPY_VERSION, board_id = ( - ctx.obj["backend"].get_circuitpython_version(device_path) + ctx.obj["backend"].get_circuitpython_version() if board_id is None or cpy_version is None else (cpy_version, board_id) ) @@ -1098,9 +1097,7 @@ def freeze(ctx, requirement): # pragma: no cover device. Option -r saves output to requirements.txt file """ logger.info("Freeze") - modules = find_modules( - ctx.obj["backend"], ctx.obj["DEVICE_PATH"], get_bundles_list() - ) + modules = find_modules(ctx.obj["backend"], get_bundles_list()) if modules: output = [] for module in modules: @@ -1133,9 +1130,7 @@ def list_cli(ctx): # pragma: no cover modules = [ m.row - for m in find_modules( - ctx.obj["backend"], ctx.obj["DEVICE_PATH"], get_bundles_list() - ) + for m in find_modules(ctx.obj["backend"], get_bundles_list()) if m.outofdate ] if modules: @@ -1213,25 +1208,21 @@ def install(ctx, modules, pyext, requirement, auto, auto_file): # pragma: no co # pass a local file with "./" or "../" is_relative = auto_file.split(os.sep)[0] in [os.path.curdir, os.path.pardir] if not os.path.isabs(auto_file) and not is_relative: - if not ctx.obj["using_webworkflow"]: - auto_file = os.path.join(ctx.obj["DEVICE_PATH"], auto_file or "code.py") - else: - auto_file = os.path.join( - ctx.obj["DEVICE_PATH"], "fs", auto_file or "code.py" - ) - if not os.path.isfile(auto_file) and not ctx.obj["using_webworkflow"]: - click.secho(f"Auto file not found: {auto_file}", fg="red") - sys.exit(1) + auto_file = ctx.obj["backend"].get_file_path(auto_file or "code.py") auto_file_path = ctx.obj["backend"].get_auto_file_path(auto_file) + if not os.path.isfile(auto_file_path): + click.secho(f"Auto file not found: {auto_file}", fg="red") + sys.exit(1) + requested_installs = libraries_from_code_py(auto_file_path, mod_names) else: requested_installs = modules requested_installs = sorted(set(requested_installs)) click.echo(f"Searching for dependencies for: {requested_installs}") to_install = get_dependencies(requested_installs, mod_names=mod_names) - device_modules = ctx.obj["backend"].get_device_versions(ctx.obj["DEVICE_PATH"]) + device_modules = ctx.obj["backend"].get_device_versions() if to_install is not None: to_install = sorted(to_install) click.echo(f"Ready to install: {to_install}\n") @@ -1276,7 +1267,7 @@ def uninstall(ctx, module): # pragma: no cover """ device_path = ctx.obj["DEVICE_PATH"] for name in module: - device_modules = ctx.obj["backend"].get_device_versions(device_path) + device_modules = ctx.obj["backend"].get_device_versions() name = name.lower() mod_names = {} for module_item, metadata in device_modules.items(): @@ -1314,11 +1305,7 @@ def update(ctx, update_all): # pragma: no cover logger.info("Update") # Grab out of date modules. modules = [ - m - for m in find_modules( - ctx.obj["backend"], ctx.obj["DEVICE_PATH"], get_bundles_list() - ) - if m.outofdate + m for m in find_modules(ctx.obj["backend"], get_bundles_list()) if m.outofdate ] if modules: click.echo("Found {} module[s] needing update.".format(len(modules))) diff --git a/circup/backends.py b/circup/backends.py index bda7afc..457ed8b 100644 --- a/circup/backends.py +++ b/circup/backends.py @@ -30,10 +30,11 @@ class Backend: """ def __init__(self, logger): + self.device_location = None self.LIB_DIR_PATH = None self.logger = logger - def get_circuitpython_version(self, device_location): + def get_circuitpython_version(self): """ Must be overridden by subclass for implementation! @@ -61,7 +62,7 @@ def get_modules(self, device_url): """ return self._get_modules(device_url) - def get_device_versions(self, device_url): + def get_device_versions(self): """ Returns a dictionary of metadata from modules on the connected device. @@ -69,7 +70,7 @@ def get_device_versions(self, device_url): :return: A dictionary of metadata about the modules available on the connected device. """ - return self.get_modules(os.path.join(device_url, self.LIB_DIR_PATH)) + return self.get_modules(os.path.join(self.device_location, self.LIB_DIR_PATH)) def _create_library_directory(self, device_path, library_path): """ @@ -150,6 +151,12 @@ def update(self, module): """ raise NotImplementedError + def get_file_path(self, filename): + """ + To be overridden by subclass + """ + raise NotImplementedError + class WebBackend(Backend): """ @@ -165,29 +172,42 @@ def __init__(self, host, password, logger): self.library_path = self.device_location + "/" + self.LIB_DIR_PATH - def install_file_http(self, source, target): + def install_file_http(self, source): """ Install file to device using web workflow. :param source source file. - :param target destination URL. Should have password embedded. """ + + target = ( + self.device_location + + "/" + + self.LIB_DIR_PATH + + source.split(os.path.sep)[-1] + ) url = urlparse(target) auth = HTTPBasicAuth("", url.password) print(f"target: {target}") + print(f"source: {source}") with open(source, "rb") as fp: r = requests.put(target, fp.read(), auth=auth) r.raise_for_status() - def install_dir_http(self, source, target): + def install_dir_http(self, source): """ Install directory to device using web workflow. :param source source directory. - :param target destination URL. Should have password embedded. """ + target = ( + self.device_location + + "/" + + self.LIB_DIR_PATH + + source.split(os.path.sep)[-1] + ) url = urlparse(target) auth = HTTPBasicAuth("", url.password) print(f"target: {target}") + print(f"source: {source}") # Create the top level directory. r = requests.put(target + ("/" if not target.endswith("/") else ""), auth=auth) @@ -209,21 +229,21 @@ def install_dir_http(self, source, target): r = requests.put(target + rel_path + "/" + name, auth=auth) r.raise_for_status() - def get_circuitpython_version(self, url): + def get_circuitpython_version(self): """ Returns the version number of CircuitPython running on the board connected via ``device_path``, along with the board ID. This is obtained using RESTful API from the /cp/version.json URL. - :param str url: board URL. :return: A tuple with the version string for CircuitPython and the board ID string. """ # pylint: disable=arguments-renamed - r = requests.get(url + "/cp/version.json") + r = requests.get(self.device_location + "/cp/version.json") # pylint: disable=no-member if r.status_code != requests.codes.ok: click.secho( - f" Unable to get version from {url}: {r.status_code}", fg="red" + f" Unable to get version from {self.device_location}: {r.status_code}", + fg="red", ) sys.exit(1) # pylint: enable=no-member @@ -345,9 +365,7 @@ def _install_module_mpy(self, bundle, metadata): if not module_name: # Must be a directory based module. module_name = os.path.basename(os.path.dirname(metadata["path"])) - major_version = self.get_circuitpython_version(self.device_location)[0].split( - "." - )[0] + major_version = self.get_circuitpython_version()[0].split(".")[0] bundle_platform = "{}mpy".format(major_version) bundle_path = os.path.join(bundle.lib_dir(bundle_platform), module_name) if os.path.isdir(bundle_path): @@ -355,13 +373,13 @@ def _install_module_mpy(self, bundle, metadata): print(f"456 library_path: {library_path}") print(f"456 module_name: {module_name}") - self.install_dir_http(bundle_path, library_path + module_name) + self.install_dir_http(bundle_path) elif os.path.isfile(bundle_path): target = os.path.basename(bundle_path) print(f"123 library_path: {library_path}") print(f"123 target: {target}") - self.install_file_http(bundle_path, library_path + target) + self.install_file_http(bundle_path) else: raise IOError("Cannot find compiled version of module.") @@ -376,11 +394,11 @@ def _install_module_py(self, metadata): source_path = metadata["path"] # Path to Python source version. if os.path.isdir(source_path): target = os.path.basename(os.path.dirname(source_path)) - self.install_dir_http(source_path, self.library_path + target) + self.install_dir_http(source_path) else: target = os.path.basename(source_path) - self.install_file_http(source_path, self.library_path + target) + self.install_file_http(source_path) def get_auto_file_path(self, auto_file_path): url = auto_file_path @@ -389,7 +407,6 @@ def get_auto_file_path(self, auto_file_path): r.raise_for_status() with open(LOCAL_CODE_PY_COPY, "w", encoding="utf-8") as f: f.write(r.text) - LOCAL_CODE_PY_COPY return LOCAL_CODE_PY_COPY def uninstall(self, device_path, module_path): @@ -416,14 +433,20 @@ def _update_http(self, module): """ if module.file: # Copy the file (will overwrite). - self.install_file_http(module.bundle_path, module.path) + self.install_file_http(module.bundle_path) else: # Delete the directory (recursive) first. url = urlparse(module.path) auth = HTTPBasicAuth("", url.password) r = requests.delete(module.path, auth=auth) r.raise_for_status() - self.install_dir_http(module.bundle_path, module.path) + self.install_dir_http(module.bundle_path) + + def get_file_path(self, filename): + """ + retuns the full path on the device to a given file name. + """ + return os.path.join(self.device_location, "fs", filename) class DiskBackend(Backend): @@ -437,7 +460,7 @@ def __init__(self, device_location, logger): self.device_location = device_location self.library_path = self.device_location + "/" + self.LIB_DIR_PATH - def get_circuitpython_version(self, device_location): + def get_circuitpython_version(self): """ Returns the version number of CircuitPython running on the board connected via ``device_path``, along with the board ID. This is obtained from the @@ -450,12 +473,13 @@ def get_circuitpython_version(self, device_location): Board ID:raspberry_pi_pico - :param str device_path: The path to the connected board. :return: A tuple with the version string for CircuitPython and the board ID string. """ try: with open( - os.path.join(device_location, "boot_out.txt"), "r", encoding="utf-8" + os.path.join(self.device_location, "boot_out.txt"), + "r", + encoding="utf-8", ) as boot: version_line = boot.readline() circuit_python = version_line.split(";")[0].split(" ")[-3] @@ -499,9 +523,7 @@ def _install_module_mpy(self, bundle, metadata): # Must be a directory based module. module_name = os.path.basename(os.path.dirname(metadata["path"])) - major_version = self.get_circuitpython_version(self.device_location)[0].split( - "." - )[0] + major_version = self.get_circuitpython_version()[0].split(".")[0] bundle_platform = "{}mpy".format(major_version) bundle_path = os.path.join(bundle.lib_dir(bundle_platform), module_name) if os.path.isdir(bundle_path): @@ -575,3 +597,9 @@ def _update_file(self, module): # Delete and copy file. os.remove(module.path) shutil.copyfile(module.bundle_path, module.path) + + def get_file_path(self, filename): + """ + retuns the full path on the device to a given file name. + """ + return os.path.join(self.device_location, filename) From 56c1ddab31f2e256e0294fd1c30293bf65f1aba9 Mon Sep 17 00:00:00 2001 From: foamyguy Date: Mon, 11 Dec 2023 16:52:13 -0600 Subject: [PATCH 28/63] allow mock version_info in DiskBackend --- circup/backends.py | 50 +++++++++++++++++++++++++--------------------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/circup/backends.py b/circup/backends.py index 457ed8b..f53260b 100644 --- a/circup/backends.py +++ b/circup/backends.py @@ -454,11 +454,12 @@ class DiskBackend(Backend): Backend for interacting with a device via USB Workflow """ - def __init__(self, device_location, logger): + def __init__(self, device_location, logger, version_info=None): super().__init__(logger) self.LIB_DIR_PATH = "lib" self.device_location = device_location self.library_path = self.device_location + "/" + self.LIB_DIR_PATH + self.version_info = version_info def get_circuitpython_version(self): """ @@ -475,28 +476,31 @@ def get_circuitpython_version(self): :return: A tuple with the version string for CircuitPython and the board ID string. """ - try: - with open( - os.path.join(self.device_location, "boot_out.txt"), - "r", - encoding="utf-8", - ) as boot: - version_line = boot.readline() - circuit_python = version_line.split(";")[0].split(" ")[-3] - board_line = boot.readline() - if board_line.startswith("Board ID:"): - board_id = board_line[9:].strip() - else: - board_id = "" - except FileNotFoundError: - click.secho( - "Missing file boot_out.txt on the device: wrong path or drive corrupted.", - fg="red", - ) - self.logger.error("boot_out.txt not found.") - sys.exit(1) - - return circuit_python, board_id + if not self.version_info: + try: + with open( + os.path.join(self.device_location, "boot_out.txt"), + "r", + encoding="utf-8", + ) as boot: + version_line = boot.readline() + circuit_python = version_line.split(";")[0].split(" ")[-3] + board_line = boot.readline() + if board_line.startswith("Board ID:"): + board_id = board_line[9:].strip() + else: + board_id = "" + except FileNotFoundError: + click.secho( + "Missing file boot_out.txt on the device: wrong path or drive corrupted.", + fg="red", + ) + self.logger.error("boot_out.txt not found.") + sys.exit(1) + + return circuit_python, board_id + else: + return self.version_info def _get_modules(self, device_lib_path): """ From a96fd8e84a50e719a0ad221d45ea5ad4bc63632c Mon Sep 17 00:00:00 2001 From: foamyguy Date: Tue, 12 Dec 2023 19:33:35 -0600 Subject: [PATCH 29/63] refactor Module() not to take path. is_deice_present() for DiskBackend --- circup/__init__.py | 32 +++++++++++++++++++++++++------- circup/backends.py | 17 +++++++++++++++++ 2 files changed, 42 insertions(+), 7 deletions(-) diff --git a/circup/__init__.py b/circup/__init__.py index b8303f8..714adf4 100644 --- a/circup/__init__.py +++ b/circup/__init__.py @@ -237,7 +237,15 @@ class Module: # pylint: disable=too-many-arguments def __init__( - self, path, repo, device_version, bundle_version, mpy, bundle, compatibility + self, + name, + backend, + repo, + device_version, + bundle_version, + mpy, + bundle, + compatibility, ): """ The ``self.file`` and ``self.name`` attributes are constructed from @@ -245,8 +253,8 @@ def __init__( resulting self.file value will be None, and the name will be the basename of the directory path. - :param str path: The path or URL to the module on the connected - CIRCUITPYTHON device. + :param str name: The file name of the module. + :param Backend backend: The backend that the module is on. :param str repo: The URL of the Git repository for this module. :param str device_version: The semver value for the version on device. :param str bundle_version: The semver value for the version in bundle. @@ -254,8 +262,10 @@ def __init__( :param Bundle bundle: Bundle object where the module is located. :param (str,str) compatibility: Min and max versions of CP compatible with the mpy. """ - self.path = path - url = urlparse(path) + self.name = name + self.backend = backend + self.path = os.path.join(backend.library_path, name) + url = urlparse(self.path) if url.scheme == "http": if url.path.endswith(".py") or url.path.endswith(".mpy"): self.file = os.path.basename(url.path) @@ -268,7 +278,7 @@ def __init__( else: if os.path.isfile(self.path): # Single file module. - self.file = os.path.basename(path) + self.file = os.path.basename(self.path) self.name = self.file.replace(".py", "").replace(".mpy", "") else: # Directory based module. @@ -593,8 +603,14 @@ def find_modules(backend, bundles_list): bundle_version = bundle_metadata.get("__version__") mpy = device_metadata["mpy"] compatibility = device_metadata.get("compatibility", (None, None)) + module_name = ( + path.split(os.sep)[-1] + if not path.endswith(os.sep) + else path[:-1].split(os.sep)[-1] + os.sep + ) m = Module( - path, + module_name, + backend, repo, device_version, bundle_version, @@ -995,6 +1011,7 @@ def main(ctx, verbose, path, host, password, board_id, cpy_version): # pragma: """ ctx.ensure_object(dict) device_path = get_device_path(host, password, path) + using_webworkflow = "host" in ctx.params.keys() and ctx.params["host"] is not None if using_webworkflow: @@ -1002,6 +1019,7 @@ def main(ctx, verbose, path, host, password, board_id, cpy_version): # pragma: else: ctx.obj["backend"] = DiskBackend(device_path, logger) + print(f'device is present ? {ctx.obj["backend"].is_device_present()}') if verbose: # Configure additional logging to stdout. global VERBOSE diff --git a/circup/backends.py b/circup/backends.py index f53260b..443b264 100644 --- a/circup/backends.py +++ b/circup/backends.py @@ -157,6 +157,12 @@ def get_file_path(self, filename): """ raise NotImplementedError + def is_device_present(self): + """ + To be overriden by subclass + """ + raise NotImplementedError + class WebBackend(Backend): """ @@ -455,6 +461,11 @@ class DiskBackend(Backend): """ def __init__(self, device_location, logger, version_info=None): + if device_location is None: + raise ValueError( + "Auto locating USB Disk based device failed. Please specify --path argument or ensure your device " + "is connected and mounted under the name CIRCUITPY." + ) super().__init__(logger) self.LIB_DIR_PATH = "lib" self.device_location = device_location @@ -607,3 +618,9 @@ def get_file_path(self, filename): retuns the full path on the device to a given file name. """ return os.path.join(self.device_location, filename) + + def is_device_present(self): + """ + To be overriden by subclass + """ + return os.path.exists(self.device_location) From 31270a0286a216141f54d1ea620957477e91a40d Mon Sep 17 00:00:00 2001 From: foamyguy Date: Wed, 13 Dec 2023 18:17:36 -0600 Subject: [PATCH 30/63] is_deice_present() for WebBackend. pylint fixes. equalize behavior when device is not present. --- circup/__init__.py | 10 ++++++---- circup/backends.py | 42 ++++++++++++++++++++++++++++-------------- circup/shared.py | 1 + 3 files changed, 35 insertions(+), 18 deletions(-) diff --git a/circup/__init__.py b/circup/__init__.py index 714adf4..157b729 100644 --- a/circup/__init__.py +++ b/circup/__init__.py @@ -954,7 +954,7 @@ def libraries_from_code_py(code_py, mod_names): try: found_imports = findimports.find_imports(code_py) except Exception as ex: # broad exception because anything could go wrong - self.logger.exception(ex) + logger.exception(ex) click.secho('Unable to read the auto file: "{}"'.format(str(ex)), fg="red") sys.exit(2) # pylint: enable=broad-except @@ -1017,9 +1017,11 @@ def main(ctx, verbose, path, host, password, board_id, cpy_version): # pragma: if using_webworkflow: ctx.obj["backend"] = WebBackend(host=host, password=password, logger=logger) else: - ctx.obj["backend"] = DiskBackend(device_path, logger) + try: + ctx.obj["backend"] = DiskBackend(device_path, logger) + except ValueError as e: + print(e) - print(f'device is present ? {ctx.obj["backend"].is_device_present()}') if verbose: # Configure additional logging to stdout. global VERBOSE @@ -1046,7 +1048,7 @@ def main(ctx, verbose, path, host, password, board_id, cpy_version): # pragma: "https://github.com/adafruit/circuitpython/releases/latest" ) global CPY_VERSION - if device_path is None: + if device_path is None or not ctx.obj["backend"].is_device_present(): click.secho("Could not find a connected CircuitPython device.", fg="red") sys.exit(1) else: diff --git a/circup/backends.py b/circup/backends.py index 443b264..4b4c51a 100644 --- a/circup/backends.py +++ b/circup/backends.py @@ -10,9 +10,7 @@ import sys import tempfile from urllib.parse import urlparse - import click -import findimports import requests from requests.auth import HTTPBasicAuth @@ -133,11 +131,11 @@ def install_module( else: click.echo("Unknown module named, '{}'.".format(name)) - def libraries_from_imports(self, code_py, mod_names): - """ - To be overridden by subclass - """ - raise NotImplementedError + # def libraries_from_imports(self, code_py, mod_names): + # """ + # To be overridden by subclass + # """ + # raise NotImplementedError def uninstall(self, device_path, module_path): """ @@ -399,14 +397,16 @@ def _install_module_py(self, metadata): source_path = metadata["path"] # Path to Python source version. if os.path.isdir(source_path): - target = os.path.basename(os.path.dirname(source_path)) self.install_dir_http(source_path) else: - target = os.path.basename(source_path) self.install_file_http(source_path) def get_auto_file_path(self, auto_file_path): + """ + Make a local temp copy of the --auto file from the device. + Returns the path to the local copy. + """ url = auto_file_path auth = HTTPBasicAuth("", self.password) r = requests.get(url, auth=auth) @@ -454,6 +454,16 @@ def get_file_path(self, filename): """ return os.path.join(self.device_location, "fs", filename) + def is_device_present(self): + """ + returns True if the device is currently connected + """ + try: + _ = self.get_device_versions() + return True + except requests.exceptions.ConnectionError: + return False + class DiskBackend(Backend): """ @@ -463,7 +473,8 @@ class DiskBackend(Backend): def __init__(self, device_location, logger, version_info=None): if device_location is None: raise ValueError( - "Auto locating USB Disk based device failed. Please specify --path argument or ensure your device " + "Auto locating USB Disk based device failed. " + "Please specify --path argument or ensure your device " "is connected and mounted under the name CIRCUITPY." ) super().__init__(logger) @@ -510,8 +521,8 @@ def get_circuitpython_version(self): sys.exit(1) return circuit_python, board_id - else: - return self.version_info + + return self.version_info def _get_modules(self, device_lib_path): """ @@ -573,6 +584,9 @@ def _install_module_py(self, metadata): shutil.copyfile(source_path, target_path) def get_auto_file_path(self, auto_file_path): + """ + Returns the path on the device to the file to be read for --auto. + """ return auto_file_path def uninstall(self, device_path, module_path): @@ -615,12 +629,12 @@ def _update_file(self, module): def get_file_path(self, filename): """ - retuns the full path on the device to a given file name. + returns the full path on the device to a given file name. """ return os.path.join(self.device_location, filename) def is_device_present(self): """ - To be overriden by subclass + returns True if the device is currently connected """ return os.path.exists(self.device_location) diff --git a/circup/shared.py b/circup/shared.py index 786c0ca..0b37a23 100644 --- a/circup/shared.py +++ b/circup/shared.py @@ -79,6 +79,7 @@ def extract_metadata(path, logger): :param str path: The path to the file containing the metadata. :return: The dunder based metadata found in the file, as a dictionary. """ + # pylint: disable=too-many-locals result = {} logger.info("%s", path) if path.endswith(".py"): From 2d0739005aed5620a8a8816b8db81b966cc9fd32 Mon Sep 17 00:00:00 2001 From: foamyguy Date: Wed, 13 Dec 2023 18:35:43 -0600 Subject: [PATCH 31/63] parse_boot_out helper. change DiskBackend() to take boot_out string instead of version info tuple --- circup/backends.py | 40 ++++++++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/circup/backends.py b/circup/backends.py index 4b4c51a..72bcd34 100644 --- a/circup/backends.py +++ b/circup/backends.py @@ -161,6 +161,22 @@ def is_device_present(self): """ raise NotImplementedError + @staticmethod + def parse_boot_out_file(boot_out_contents): + """ + Parse the contents of boot_out.txt + Returns: circuitpython version and board id + """ + lines = boot_out_contents.split("\n") + version_line = lines[0] + circuit_python = version_line.split(";")[0].split(" ")[-3] + board_line = lines[1] + if board_line.startswith("Board ID:"): + board_id = board_line[9:].strip() + else: + board_id = "" + return circuit_python, board_id + class WebBackend(Backend): """ @@ -468,9 +484,14 @@ def is_device_present(self): class DiskBackend(Backend): """ Backend for interacting with a device via USB Workflow + + :param String device_location: Path to the device + :param logger: logger to use for outputting messages + :param String boot_out: Optional mock contents of a boot_out.txt file + to use for version information. """ - def __init__(self, device_location, logger, version_info=None): + def __init__(self, device_location, logger, boot_out=None): if device_location is None: raise ValueError( "Auto locating USB Disk based device failed. " @@ -481,7 +502,9 @@ def __init__(self, device_location, logger, version_info=None): self.LIB_DIR_PATH = "lib" self.device_location = device_location self.library_path = self.device_location + "/" + self.LIB_DIR_PATH - self.version_info = version_info + self.version_info = None + if boot_out is not None: + self.version_info = self.parse_boot_out_file(boot_out) def get_circuitpython_version(self): """ @@ -505,13 +528,10 @@ def get_circuitpython_version(self): "r", encoding="utf-8", ) as boot: - version_line = boot.readline() - circuit_python = version_line.split(";")[0].split(" ")[-3] - board_line = boot.readline() - if board_line.startswith("Board ID:"): - board_id = board_line[9:].strip() - else: - board_id = "" + boot_out_contents = boot.read() + circuit_python, board_id = self.parse_boot_out_file( + boot_out_contents + ) except FileNotFoundError: click.secho( "Missing file boot_out.txt on the device: wrong path or drive corrupted.", @@ -519,7 +539,7 @@ def get_circuitpython_version(self): ) self.logger.error("boot_out.txt not found.") sys.exit(1) - + print((circuit_python, board_id)) return circuit_python, board_id return self.version_info From 1f9bee3ffcf990446b06d7cc88d21d16345a272c Mon Sep 17 00:00:00 2001 From: foamyguy Date: Mon, 18 Dec 2023 10:43:34 -0600 Subject: [PATCH 32/63] black format b4 pylint. fix tests tests to passing --- .pre-commit-config.yaml | 10 +- circup/__init__.py | 2 +- tests/mock_device/boot_out.txt | 5 +- tests/test_circup.py | 310 +++++++++++++++++++++------------ 4 files changed, 211 insertions(+), 116 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fa3d605..a76c7f4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,6 +3,11 @@ # SPDX-License-Identifier: Unlicense repos: +- repo: https://github.com/python/black + rev: 22.3.0 + hooks: + - id: black + exclude: "^tests/bad_python.py$" - repo: https://github.com/pycqa/pylint rev: v2.15.5 hooks: @@ -16,11 +21,6 @@ repos: name: lint (code) types: [python] exclude: "^(docs/|examples/|setup.py$|tests/bad_python.py$)" -- repo: https://github.com/python/black - rev: 22.3.0 - hooks: - - id: black - exclude: "^tests/bad_python.py$" - repo: https://github.com/fsfe/reuse-tool rev: v0.14.0 hooks: diff --git a/circup/__init__.py b/circup/__init__.py index 157b729..8819f1a 100644 --- a/circup/__init__.py +++ b/circup/__init__.py @@ -283,7 +283,7 @@ def __init__( else: # Directory based module. self.file = None - self.name = os.path.basename(os.path.dirname(self.path)) + self.path = os.path.join(backend.library_path, name, "") self.repo = repo self.device_version = device_version self.bundle_version = bundle_version diff --git a/tests/mock_device/boot_out.txt b/tests/mock_device/boot_out.txt index ee14d1c..43d58ee 100644 --- a/tests/mock_device/boot_out.txt +++ b/tests/mock_device/boot_out.txt @@ -1,2 +1,3 @@ -Adafruit CircuitPython 4.1.0 on 2019-08-02; -Adafruit CircuitPlayground Express with samd21g18 +Adafruit CircuitPython 4.1.0 on 2019-08-02; Adafruit CircuitPlayground Express with samd21g18 +Board ID:this_is_a_board +UID:AAAABBBBCCCC diff --git a/tests/test_circup.py b/tests/test_circup.py index f34e651..fa561c1 100644 --- a/tests/test_circup.py +++ b/tests/test_circup.py @@ -31,14 +31,12 @@ import json import pathlib from unittest import mock - - from click.testing import CliRunner import pytest import requests import circup - +from circup import DiskBackend TEST_BUNDLE_CONFIG_JSON = "tests/test_bundle_config.json" with open(TEST_BUNDLE_CONFIG_JSON, "rb") as tbc: @@ -211,18 +209,29 @@ def test_Module_init_file_module(): Ensure the Module instance is set up as expected and logged, as if for a single file Python module. """ - path = os.path.join("foo", "bar", "baz", "local_module.py") + name = "local_module.py" + path = os.path.join("mock_device", "lib", name) repo = "https://github.com/adafruit/SomeLibrary.git" device_version = "1.2.3" bundle_version = "3.2.1" + with mock.patch("circup.logger.info") as mock_logger, mock.patch( "circup.os.path.isfile", return_value=True ), mock.patch("circup.CPY_VERSION", "4.1.2"), mock.patch( - "circup.Bundle.lib_dir", return_value="tests" + "circup.Bundle.lib_dir", + return_value="tests", ): + backend = DiskBackend("mock_device", mock_logger) bundle = circup.Bundle(TEST_BUNDLE_NAME) m = circup.Module( - path, repo, device_version, bundle_version, False, bundle, (None, None) + name, + backend, + repo, + device_version, + bundle_version, + False, + bundle, + (None, None), ) mock_logger.assert_called_once_with(m) assert m.path == path @@ -240,7 +249,8 @@ def test_Module_init_directory_module(): Ensure the Module instance is set up as expected and logged, as if for a directory based Python module. """ - path = os.path.join("foo", "bar", "dir_module", "") + name = "dir_module" + path = os.path.join("mock_device", "lib", f"{name}", "") repo = "https://github.com/adafruit/SomeLibrary.git" device_version = "1.2.3" bundle_version = "3.2.1" @@ -252,10 +262,19 @@ def test_Module_init_directory_module(): ), mock.patch( "circup.Bundle.lib_dir", return_value="tests" ): + backend = DiskBackend("mock_device", mock_logger) bundle = circup.Bundle(TEST_BUNDLE_NAME) m = circup.Module( - path, repo, device_version, bundle_version, mpy, bundle, (None, None) + name, + backend, + repo, + device_version, + bundle_version, + mpy, + bundle, + (None, None), ) + print(f"NAMENAME {m.name}") mock_logger.assert_called_once_with(m) assert m.path == path assert m.file is None @@ -274,15 +293,23 @@ def test_Module_outofdate(): out of date. """ bundle = circup.Bundle(TEST_BUNDLE_NAME) - path = os.path.join("foo", "bar", "baz", "module.py") + name = "module.py" repo = "https://github.com/adafruit/SomeLibrary.git" - m1 = circup.Module(path, repo, "1.2.3", "3.2.1", False, bundle, (None, None)) - m2 = circup.Module(path, repo, "1.2.3", "1.2.3", False, bundle, (None, None)) - # shouldn't happen! - m3 = circup.Module(path, repo, "3.2.1", "1.2.3", False, bundle, (None, None)) - assert m1.outofdate is True - assert m2.outofdate is False - assert m3.outofdate is False + with mock.patch("circup.logger.info") as mock_logger: + backend = DiskBackend("mock_device", mock_logger) + m1 = circup.Module( + name, backend, repo, "1.2.3", "3.2.1", False, bundle, (None, None) + ) + m2 = circup.Module( + name, backend, repo, "1.2.3", "1.2.3", False, bundle, (None, None) + ) + # shouldn't happen! + m3 = circup.Module( + name, backend, repo, "3.2.1", "1.2.3", False, bundle, (None, None) + ) + assert m1.outofdate is True + assert m2.outofdate is False + assert m3.outofdate is False def test_Module_outofdate_bad_versions(): @@ -292,14 +319,25 @@ def test_Module_outofdate_bad_versions(): this problem). Such a problem should be logged. """ bundle = circup.Bundle(TEST_BUNDLE_NAME) - path = os.path.join("foo", "bar", "baz", "module.py") + name = "module.py" + repo = "https://github.com/adafruit/SomeLibrary.git" device_version = "hello" bundle_version = "3.2.1" - m = circup.Module( - path, repo, device_version, bundle_version, False, bundle, (None, None) - ) + with mock.patch("circup.logger.warning") as mock_logger: + backend = DiskBackend("mock_device", mock_logger) + m = circup.Module( + name, + backend, + repo, + device_version, + bundle_version, + False, + bundle, + (None, None), + ) + assert m.outofdate is True assert mock_logger.call_count == 2 @@ -310,16 +348,21 @@ def test_Module_mpy_mismatch(): boolean value to correctly indicate if the referenced module is, in fact, out of date. """ - path = os.path.join("foo", "bar", "baz", "module.mpy") + name = "module.py" repo = "https://github.com/adafruit/SomeLibrary.git" - with mock.patch("circup.CPY_VERSION", "8.0.0"): + with mock.patch("circup.CPY_VERSION", "8.0.0"), mock.patch( + "circup.logger.warning" + ) as mock_logger: + backend = DiskBackend("mock_device", mock_logger) bundle = circup.Bundle(TEST_BUNDLE_NAME) - m1 = circup.Module(path, repo, "1.2.3", "1.2.3", True, bundle, (None, None)) + m1 = circup.Module( + name, backend, repo, "1.2.3", "1.2.3", True, bundle, (None, None) + ) m2 = circup.Module( - path, repo, "1.2.3", "1.2.3", True, bundle, ("7.0.0-alpha.1", None) + name, backend, repo, "1.2.3", "1.2.3", True, bundle, ("7.0.0-alpha.1", None) ) m3 = circup.Module( - path, repo, "1.2.3", "1.2.3", True, bundle, (None, "7.0.0-alpha.1") + name, backend, repo, "1.2.3", "1.2.3", True, bundle, (None, "7.0.0-alpha.1") ) with mock.patch("circup.CPY_VERSION", "6.2.0"): assert m1.mpy_mismatch is False @@ -345,14 +388,23 @@ def test_Module_major_update_bad_versions(): Such a problem should be logged. """ bundle = circup.Bundle(TEST_BUNDLE_NAME) - path = os.path.join("foo", "bar", "baz", "module.py") + name = "module.py" repo = "https://github.com/adafruit/SomeLibrary.git" device_version = "1.2.3" bundle_version = "version-3" - m = circup.Module( - path, repo, device_version, bundle_version, False, bundle, (None, None) - ) + with mock.patch("circup.logger.warning") as mock_logger: + backend = DiskBackend("mock_device", mock_logger) + m = circup.Module( + name, + backend, + repo, + device_version, + bundle_version, + False, + bundle, + (None, None), + ) assert m.major_update is True assert mock_logger.call_count == 2 @@ -363,16 +415,23 @@ def test_Module_row(): a table of version-related results. """ bundle = circup.Bundle(TEST_BUNDLE_NAME) - path = os.path.join("foo", "bar", "baz", "module.py") + name = "module.py" repo = "https://github.com/adafruit/SomeLibrary.git" with mock.patch("circup.os.path.isfile", return_value=True), mock.patch( "circup.CPY_VERSION", "8.0.0" - ): - m = circup.Module(path, repo, "1.2.3", None, False, bundle, (None, None)) + ), mock.patch("circup.logger.warning") as mock_logger: + backend = DiskBackend("mock_device", mock_logger) + m = circup.Module( + name, backend, repo, "1.2.3", None, False, bundle, (None, None) + ) assert m.row == ("module", "1.2.3", "unknown", "Major Version") - m = circup.Module(path, repo, "1.2.3", "1.3.4", False, bundle, (None, None)) + m = circup.Module( + name, backend, repo, "1.2.3", "1.3.4", False, bundle, (None, None) + ) assert m.row == ("module", "1.2.3", "1.3.4", "Minor Version") - m = circup.Module(path, repo, "1.2.3", "1.2.3", True, bundle, ("9.0.0", None)) + m = circup.Module( + name, backend, repo, "1.2.3", "1.2.3", True, bundle, ("9.0.0", None) + ) assert m.row == ("module", "1.2.3", "1.2.3", "MPY Format") @@ -382,17 +441,24 @@ def test_Module_update_dir(): update the module on the connected device. """ bundle = circup.Bundle(TEST_BUNDLE_NAME) - path = os.path.join("foo", "bar", "baz", "module.py") + name = "adafruit_waveform" repo = "https://github.com/adafruit/SomeLibrary.git" device_version = "1.2.3" bundle_version = None - m = circup.Module( - path, repo, device_version, bundle_version, False, bundle, (None, None) - ) - backend = circup.DiskBackend() - with mock.patch("circup.shutil") as mock_shutil, mock.patch( + with mock.patch("circup.backends.shutil") as mock_shutil, mock.patch( "circup.os.path.isdir", return_value=True - ): + ), mock.patch("circup.logger.warning") as mock_logger: + backend = DiskBackend("mock_device", mock_logger) + m = circup.Module( + name, + backend, + repo, + device_version, + bundle_version, + False, + bundle, + (None, None), + ) backend.update(m) mock_shutil.rmtree.assert_called_once_with(m.path, ignore_errors=True) mock_shutil.copytree.assert_called_once_with(m.bundle_path, m.path) @@ -404,17 +470,30 @@ def test_Module_update_file(): update the module on the connected device. """ bundle = circup.Bundle(TEST_BUNDLE_NAME) - path = os.path.join("foo", "bar", "baz", "module.py") + name = "colorsys.py" + # path = os.path.join("foo", "bar", "baz", "module.py") repo = "https://github.com/adafruit/SomeLibrary.git" device_version = "1.2.3" bundle_version = None - m = circup.Module( - path, repo, device_version, bundle_version, False, bundle, (None, None) - ) - backend = circup.DiskBackend() - with mock.patch("circup.shutil") as mock_shutil, mock.patch( + + with mock.patch("circup.backends.shutil") as mock_shutil, mock.patch( "circup.os.remove" - ) as mock_remove, mock.patch("circup.os.path.isdir", return_value=False): + ) as mock_remove, mock.patch( + "circup.os.path.isdir", return_value=False + ), mock.patch( + "circup.logger.warning" + ) as mock_logger: + backend = circup.DiskBackend("mock_device", mock_logger) + m = circup.Module( + name, + backend, + repo, + device_version, + bundle_version, + False, + bundle, + (None, None), + ) backend.update(m) mock_remove.assert_called_once_with(m.path) mock_shutil.copyfile.assert_called_once_with(m.bundle_path, m.path) @@ -424,16 +503,27 @@ def test_Module_repr(): """ Ensure the repr(dict) is returned (helps when logging). """ - path = os.path.join("foo", "bar", "baz", "local_module.py") + name = "local_module.py" + path = os.path.join("mock_device", "lib", f"{name}") repo = "https://github.com/adafruit/SomeLibrary.git" device_version = "1.2.3" bundle_version = "3.2.1" with mock.patch("circup.os.path.isfile", return_value=True), mock.patch( "circup.CPY_VERSION", "4.1.2" - ), mock.patch("circup.Bundle.lib_dir", return_value="tests"): + ), mock.patch("circup.Bundle.lib_dir", return_value="tests"), mock.patch( + "circup.logger.warning" + ) as mock_logger: bundle = circup.Bundle(TEST_BUNDLE_NAME) + backend = circup.DiskBackend("mock_device", mock_logger) m = circup.Module( - path, repo, device_version, bundle_version, False, bundle, (None, None) + name, + backend, + repo, + device_version, + bundle_version, + False, + bundle, + (None, None), ) assert repr(m) == repr( { @@ -565,8 +655,10 @@ def test_extract_metadata_python(): 'print("Hello, world!")\n' ) path = "foo.py" - with mock.patch("builtins.open", mock.mock_open(read_data=code)) as mock_open: - result = circup.extract_metadata(path) + with mock.patch( + "builtins.open", mock.mock_open(read_data=code) + ) as mock_open, mock.patch("circup.logger.warning") as mock_logger: + result = circup.extract_metadata(path, mock_logger) mock_open.assert_called_once_with(path, "r", encoding="utf-8") assert len(result) == 3 assert result["__version__"] == "1.1.4" @@ -580,10 +672,11 @@ def test_extract_metadata_byte_code_v6(): Ensure the __version__ is correctly extracted from the bytecode ".mpy" file generated from Circuitpython < 7. Version in test_module is 0.9.2 """ - result = circup.extract_metadata("tests/test_module.mpy") - assert result["__version__"] == "0.9.2" - assert result["mpy"] is True - assert result["compatibility"] == (None, "7.0.0-alpha.1") + with mock.patch("circup.logger.warning") as mock_logger: + result = circup.extract_metadata("tests/test_module.mpy", mock_logger) + assert result["__version__"] == "0.9.2" + assert result["mpy"] is True + assert result["compatibility"] == (None, "7.0.0-alpha.1") def test_extract_metadata_byte_code_v7(): @@ -591,10 +684,11 @@ def test_extract_metadata_byte_code_v7(): Ensure the __version__ is correctly extracted from the bytecode ".mpy" file generated from Circuitpython >= 7. Version in local_module_cp7 is 1.2.3 """ - result = circup.extract_metadata("tests/local_module_cp7.mpy") - assert result["__version__"] == "1.2.3" - assert result["mpy"] is True - assert result["compatibility"] == ("7.0.0-alpha.1", None) + with mock.patch("circup.logger.warning") as mock_logger: + result = circup.extract_metadata("tests/local_module_cp7.mpy", mock_logger) + assert result["__version__"] == "1.2.3" + assert result["mpy"] is True + assert result["compatibility"] == ("7.0.0-alpha.1", None) def test_find_modules(): @@ -608,19 +702,21 @@ def test_find_modules(): bundle_modules = json.load(f) with mock.patch( - "circup.USBBackend.get_device_versions", return_value=device_modules + "circup.DiskBackend.get_device_versions", return_value=device_modules ), mock.patch( "circup.get_bundle_versions", return_value=bundle_modules ), mock.patch( "circup.os.path.isfile", return_value=True - ): - backend = circup.DiskBackend() + ), mock.patch( + "circup.logger.warning" + ) as mock_logger: + backend = DiskBackend("mock_device", mock_logger) bundle = circup.Bundle(TEST_BUNDLE_NAME) bundles_list = [bundle] for module in bundle_modules: bundle_modules[module]["bundle"] = bundle - result = circup.find_modules(backend, "", bundles_list) + result = circup.find_modules(backend, bundles_list) assert len(result) == 1 assert result[0].name == "adafruit_74hc595" assert ( @@ -635,14 +731,16 @@ def test_find_modules_goes_bang(): and the utility exists with an error code of 1. """ with mock.patch( - "circup.USBBackend.get_device_versions", side_effect=Exception("BANG!") + "circup.DiskBackend.get_device_versions", side_effect=Exception("BANG!") ), mock.patch("circup.click") as mock_click, mock.patch( "circup.sys.exit" - ) as mock_exit: + ) as mock_exit, mock.patch( + "circup.logger.warning" + ) as mock_logger: bundle = circup.Bundle(TEST_BUNDLE_NAME) bundles_list = [bundle] - backend = circup.DiskBackend() - circup.find_modules(backend, "", bundles_list) + backend = DiskBackend("mock_devcie", mock_logger) + circup.find_modules(backend, bundles_list) assert mock_click.echo.call_count == 1 mock_exit.assert_called_once_with(1) @@ -658,14 +756,16 @@ def test_get_bundle_versions(): "circup.Bundle.lib_dir", return_value="foo/bar/lib" ), mock.patch( "circup.os.path.isdir", return_value=True - ): + ), mock.patch( + "circup.logger" + ) as mock_logger: bundle = circup.Bundle(TEST_BUNDLE_NAME) bundles_list = [bundle] assert circup.get_bundle_versions(bundles_list) == { "ok": {"name": "ok", "bundle": bundle} } mock_elb.assert_called_once_with(bundle) - mock_gm.assert_called_once_with("foo/bar/lib") + mock_gm.assert_called_once_with("foo/bar/lib", mock_logger) def test_get_bundle_versions_avoid_download(): @@ -677,7 +777,9 @@ def test_get_bundle_versions_avoid_download(): "circup._get_modules_file", return_value={"ok": {"name": "ok"}} ) as mock_gm, mock.patch("circup.CPY_VERSION", "4.1.2"), mock.patch( "circup.Bundle.lib_dir", return_value="foo/bar/lib" - ): + ), mock.patch( + "circup.logger" + ) as mock_logger: bundle = circup.Bundle(TEST_BUNDLE_NAME) bundles_list = [bundle] with mock.patch("circup.os.path.isdir", return_value=True): @@ -685,13 +787,13 @@ def test_get_bundle_versions_avoid_download(): "ok": {"name": "ok", "bundle": bundle} } assert mock_elb.call_count == 0 - mock_gm.assert_called_once_with("foo/bar/lib") + mock_gm.assert_called_once_with("foo/bar/lib", mock_logger) with mock.patch("circup.os.path.isdir", return_value=False): assert circup.get_bundle_versions(bundles_list, avoid_download=True) == { "ok": {"name": "ok", "bundle": bundle} } mock_elb.assert_called_once_with(bundle) - mock_gm.assert_called_with("foo/bar/lib") + mock_gm.assert_called_with("foo/bar/lib", mock_logger) def test_get_circuitpython_version(): @@ -699,39 +801,24 @@ def test_get_circuitpython_version(): Given valid content of a boot_out.txt file on a connected device, return the version number of CircuitPython running on the board. """ - device_path = "device" - data_no_id = ( - "Adafruit CircuitPython 4.1.0 on 2019-08-02; " - "Adafruit CircuitPlayground Express with samd21g18" - ) - with mock.patch("builtins.open", mock.mock_open(read_data=data_no_id)) as mock_open: - backend = circup.DiskBackend() - assert backend.get_circuitpython_version(device_path) == ("4.1.0", "") - mock_open.assert_called_once_with( - os.path.join(device_path, "boot_out.txt"), "r", encoding="utf-8" - ) - data_with_id = data_no_id + "\r\n" "Board ID:this_is_a_board" - with mock.patch( - "builtins.open", mock.mock_open(read_data=data_with_id) - ) as mock_open: - backend = circup.DiskBackend() - assert backend.get_circuitpython_version(device_path) == ( + with mock.patch("circup.logger.warning") as mock_logger: + backend = DiskBackend("tests/mock_device", mock_logger) + assert backend.get_circuitpython_version() == ( "4.1.0", "this_is_a_board", ) - mock_open.assert_called_once_with( - os.path.join(device_path, "boot_out.txt"), "r", encoding="utf-8" - ) def test_get_device_versions(): """ Ensure get_modules is called with the path for the attached device. """ - with mock.patch("circup.USBBackend.get_modules", return_value="ok") as mock_gm: - backend = circup.DiskBackend() - assert backend.get_device_versions("TESTDIR") == "ok" - mock_gm.assert_called_once_with(os.path.join("TESTDIR", "lib")) + with mock.patch( + "circup.DiskBackend.get_modules", return_value="ok" + ) as mock_gm, mock.patch("circup.logger.warning") as mock_logger: + backend = circup.DiskBackend("mock_device", mock_logger) + assert backend.get_device_versions() == "ok" + mock_gm.assert_called_once_with(os.path.join("mock_device", "lib")) def test_get_modules_empty_path(): @@ -739,8 +826,9 @@ def test_get_modules_empty_path(): Sometimes a path to a device or bundle may be empty. Ensure, if this is the case, an empty dictionary is returned. """ - backend = circup.DiskBackend() - assert backend.get_modules("") == {} + with mock.patch("circup.logger.warning") as mock_logger: + backend = circup.DiskBackend("tests/mock_device", mock_logger) + assert backend.get_modules("") == {} def test_get_modules_that_are_files(): @@ -753,8 +841,10 @@ def test_get_modules_that_are_files(): os.path.join("tests", "local_module.py"), os.path.join("tests", ".hidden_module.py"), ] - with mock.patch("circup.glob.glob", side_effect=[mods, [], []]): - backend = circup.DiskBackend() + with mock.patch("circup.glob.glob", side_effect=[mods, [], []]), mock.patch( + "circup.logger.warning" + ) as mock_logger: + backend = circup.DiskBackend("mock_device", mock_logger) result = backend.get_modules(path) assert len(result) == 1 # Hidden files are ignored. assert "local_module" in result @@ -777,8 +867,10 @@ def test_get_modules_that_are_directories(): os.path.join("tests", ".hidden_dir", ""), ] mod_files = ["tests/dir_module/my_module.py", "tests/dir_module/__init__.py"] - with mock.patch("circup.glob.glob", side_effect=[[], [], mods, mod_files, []]): - backend = circup.DiskBackend() + with mock.patch( + "circup.glob.glob", side_effect=[[], [], mods, mod_files, []] + ), mock.patch("circup.logger.warning") as mock_logger: + backend = circup.DiskBackend("mock_device", mock_logger) result = backend.get_modules(path) assert len(result) == 1 assert "dir_module" in result @@ -796,8 +888,10 @@ def test_get_modules_that_are_directories_with_no_metadata(): path = "tests" # mocked away in function. mods = [os.path.join("tests", "bad_module", "")] mod_files = ["tests/bad_module/my_module.py", "tests/bad_module/__init__.py"] - with mock.patch("circup.glob.glob", side_effect=[[], [], mods, mod_files, []]): - backend = circup.DiskBackend() + with mock.patch( + "circup.glob.glob", side_effect=[[], [], mods, mod_files, []] + ), mock.patch("circup.logger.warning") as mock_logger: + backend = circup.DiskBackend("mock_device", mock_logger) result = backend.get_modules(path) assert len(result) == 1 assert "bad_module" in result @@ -1032,8 +1126,8 @@ def test_libraries_from_imports(): "adafruit_touchscreen", ] test_file = str(pathlib.Path(__file__).parent / "import_styles.py") - backend = circup.DiskBackend() - result = backend.libraries_from_imports(test_file, mod_names) + + result = circup.libraries_from_code_py(test_file, mod_names) print(result) assert result == [ "adafruit_bus_device", From 01cceec6bfedc65d1242b6774762b6fbd6d544c8 Mon Sep 17 00:00:00 2001 From: foamyguy Date: Mon, 18 Dec 2023 10:48:32 -0600 Subject: [PATCH 33/63] remove prints --- tests/test_circup.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_circup.py b/tests/test_circup.py index fa561c1..03e2d4d 100644 --- a/tests/test_circup.py +++ b/tests/test_circup.py @@ -274,7 +274,6 @@ def test_Module_init_directory_module(): bundle, (None, None), ) - print(f"NAMENAME {m.name}") mock_logger.assert_called_once_with(m) assert m.path == path assert m.file is None @@ -1128,7 +1127,6 @@ def test_libraries_from_imports(): test_file = str(pathlib.Path(__file__).parent / "import_styles.py") result = circup.libraries_from_code_py(test_file, mod_names) - print(result) assert result == [ "adafruit_bus_device", "adafruit_button", From 6cc6a1df3996beb7cbdc103bdce4db6be14f4e2a Mon Sep 17 00:00:00 2001 From: tyeth Date: Sun, 11 Feb 2024 15:46:49 +0000 Subject: [PATCH 34/63] Tweak for Web Workflow with sub paths --- circup/__init__.py | 24 +++++++++++++++++------- circup/backends.py | 44 +++++++++++++++++++++++++++++++++----------- 2 files changed, 50 insertions(+), 18 deletions(-) diff --git a/circup/__init__.py b/circup/__init__.py index 8819f1a..ed8e9af 100644 --- a/circup/__init__.py +++ b/circup/__init__.py @@ -18,7 +18,7 @@ import tempfile import zipfile from subprocess import check_output -from urllib.parse import urlparse +from urllib.parse import urlparse, urljoin import appdirs import click @@ -264,9 +264,9 @@ def __init__( """ self.name = name self.backend = backend - self.path = os.path.join(backend.library_path, name) - url = urlparse(self.path) - if url.scheme == "http": + self.path = urljoin(backend.library_path,name, allow_fragments=False) if isinstance(backend,WebBackend) else os.path.join(backend.library_path, name) + url = urlparse(self.path,allow_fragments=False) + if str(url.scheme).lower() in ("http", "https"): if url.path.endswith(".py") or url.path.endswith(".mpy"): self.file = os.path.basename(url.path) self.name = ( @@ -274,7 +274,7 @@ def __init__( ) else: self.file = None - self.name = os.path.basename(url.path[:-1]) + self.name = os.path.basename(url.path if url.path[:-1]=='/' else url.path[:-1]) else: if os.path.isfile(self.path): # Single file module. @@ -608,6 +608,12 @@ def find_modules(backend, bundles_list): if not path.endswith(os.sep) else path[:-1].split(os.sep)[-1] + os.sep ) + print("old path module name: ", module_name) + module_name = name if not path.contains(os.sep) else module_name # should probably check for os.sep and use previous version if found + print("new path module name: ", module_name) + print("Name: ", name) + print("Path: ", path) + print("Module_name: ", module_name) m = Module( module_name, backend, @@ -618,6 +624,7 @@ def find_modules(backend, bundles_list): bundle, compatibility, ) + print("Module: ", m) result.append(m) return result except Exception as ex: @@ -1225,13 +1232,16 @@ def install(ctx, modules, pyext, requirement, auto, auto_file): # pragma: no co elif auto or auto_file: if auto_file is None: auto_file = "code.py" + print(f"Auto file: {auto_file}") # pass a local file with "./" or "../" - is_relative = auto_file.split(os.sep)[0] in [os.path.curdir, os.path.pardir] + is_relative = not isinstance(ctx.obj["backend"], WebBackend) or auto_file.split(os.sep)[0] in [os.path.curdir, os.path.pardir] + print("is rel: ", is_relative, "is abs: ", os.path.isabs(auto_file)) if not os.path.isabs(auto_file) and not is_relative: auto_file = ctx.obj["backend"].get_file_path(auto_file or "code.py") + print(f"Auto file: {auto_file}") auto_file_path = ctx.obj["backend"].get_auto_file_path(auto_file) - + print(f"Auto file path: {auto_file_path}") if not os.path.isfile(auto_file_path): click.secho(f"Auto file not found: {auto_file}", fg="red") sys.exit(1) diff --git a/circup/backends.py b/circup/backends.py index 72bcd34..3dc75c3 100644 --- a/circup/backends.py +++ b/circup/backends.py @@ -9,7 +9,7 @@ import shutil import sys import tempfile -from urllib.parse import urlparse +from urllib.parse import urlparse, urljoin import click import requests from requests.auth import HTTPBasicAuth @@ -114,7 +114,7 @@ def install_module( click.echo("'{}' is already installed.".format(name)) return - library_path = os.path.join(device_path, self.LIB_DIR_PATH) + library_path = os.path.join(device_path, self.LIB_DIR_PATH) if not isinstance(self,WebBackend) else urljoin(device_path,self.LIB_DIR_PATH) # Create the library directory first. self._create_library_directory(device_path, library_path) @@ -197,12 +197,13 @@ def install_file_http(self, source): Install file to device using web workflow. :param source source file. """ - + file_name = source.split(os.path.sep) + file_name = file_name[-2] if file_name[-1] == "" else file_name[-1] target = ( self.device_location + "/" + self.LIB_DIR_PATH - + source.split(os.path.sep)[-1] + + file_name ) url = urlparse(target) auth = HTTPBasicAuth("", url.password) @@ -218,11 +219,13 @@ def install_dir_http(self, source): Install directory to device using web workflow. :param source source directory. """ + mod_name = source.split(os.path.sep) + mod_name = mod_name[-2] if mod_name[-1] == "" else mod_name[-1] target = ( self.device_location + "/" + self.LIB_DIR_PATH - + source.split(os.path.sep)[-1] + + mod_name ) url = urlparse(target) auth = HTTPBasicAuth("", url.password) @@ -230,7 +233,7 @@ def install_dir_http(self, source): print(f"source: {source}") # Create the top level directory. - r = requests.put(target + ("/" if not target.endswith("/") else ""), auth=auth) + r = requests.put((target + "/" if target[:-1]!="/" else target), auth=auth) print(f"resp {r.content}") r.raise_for_status() @@ -241,12 +244,16 @@ def install_dir_http(self, source): rel_path = "" for name in files: with open(os.path.join(root, name), "rb") as fp: + path_to_create = urljoin( urljoin(target , rel_path + "/", allow_fragments=False) , name, allow_fragments=False) if rel_path != "" else urljoin(target , name, allow_fragments=False) + # print(f"file_path_to_create: {path_to_create}") r = requests.put( - target + rel_path + "/" + name, fp.read(), auth=auth + path_to_create, fp.read(), auth=auth ) r.raise_for_status() for name in dirs: - r = requests.put(target + rel_path + "/" + name, auth=auth) + path_to_create = urljoin( urljoin(target , rel_path + "/", allow_fragments=False) , name, allow_fragments=False) if rel_path != "" else urljoin(target , name, allow_fragments=False) + # print(f"dir_path_to_create: {path_to_create}") + r = requests.put(path_to_create, auth=auth) r.raise_for_status() def get_circuitpython_version(self): @@ -312,7 +319,10 @@ def _get_modules_http_dir_mods(self, auth, directory_mods, result, url): :param url: URL of the device. """ for dm in directory_mods: - dm_url = url + dm + "/" + if not str(urlparse(dm).scheme).lower() in ("http", "https"): + dm_url = url + dm + "/" + else: + dm_url = dm r = requests.get(dm_url, auth=auth, headers={"Accept": "application/json"}) r.raise_for_status() mpy = False @@ -353,7 +363,10 @@ def _get_modules_http_single_mods(self, auth, result, single_file_mods, url): :param url: URL of the device. """ for sfm in single_file_mods: - sfm_url = url + sfm + if not str(urlparse(sfm).scheme).lower() in ("http", "https"): + sfm_url = url + sfm + else: + sfm_url = sfm r = requests.get(sfm_url, auth=auth) r.raise_for_status() idx = sfm.rfind(".") @@ -468,7 +481,7 @@ def get_file_path(self, filename): """ retuns the full path on the device to a given file name. """ - return os.path.join(self.device_location, "fs", filename) + return urljoin( urljoin(self.device_location, "fs/", allow_fragments=False), filename, allow_fragments=False) def is_device_present(self): """ @@ -480,6 +493,15 @@ def is_device_present(self): except requests.exceptions.ConnectionError: return False + def get_device_versions(self): + """ + Returns a dictionary of metadata from modules on the connected device. + + :param str device_url: URL for the device. + :return: A dictionary of metadata about the modules available on the + connected device. + """ + return self.get_modules(urljoin(self.device_location, self.LIB_DIR_PATH)) class DiskBackend(Backend): """ From e5a20309a098d6621fbb2716c4815869c58fda56 Mon Sep 17 00:00:00 2001 From: tyeth Date: Sun, 11 Feb 2024 17:54:44 +0000 Subject: [PATCH 35/63] Dispose of request sockets + create DIRs first Saw ConnectionResetError: [WinError 10054] An existing connection was forcibly closed by the remote host --- circup/__init__.py | 13 ++-- circup/backends.py | 164 +++++++++++++++++++++++---------------------- 2 files changed, 87 insertions(+), 90 deletions(-) diff --git a/circup/__init__.py b/circup/__init__.py index ed8e9af..7fb24a0 100644 --- a/circup/__init__.py +++ b/circup/__init__.py @@ -608,12 +608,7 @@ def find_modules(backend, bundles_list): if not path.endswith(os.sep) else path[:-1].split(os.sep)[-1] + os.sep ) - print("old path module name: ", module_name) - module_name = name if not path.contains(os.sep) else module_name # should probably check for os.sep and use previous version if found - print("new path module name: ", module_name) - print("Name: ", name) - print("Path: ", path) - print("Module_name: ", module_name) + module_name = name if not path.find(os.sep) else module_name # should probably check for os.sep and use previous version if found m = Module( module_name, backend, @@ -624,7 +619,6 @@ def find_modules(backend, bundles_list): bundle, compatibility, ) - print("Module: ", m) result.append(m) return result except Exception as ex: @@ -1235,10 +1229,8 @@ def install(ctx, modules, pyext, requirement, auto, auto_file): # pragma: no co print(f"Auto file: {auto_file}") # pass a local file with "./" or "../" is_relative = not isinstance(ctx.obj["backend"], WebBackend) or auto_file.split(os.sep)[0] in [os.path.curdir, os.path.pardir] - print("is rel: ", is_relative, "is abs: ", os.path.isabs(auto_file)) if not os.path.isabs(auto_file) and not is_relative: auto_file = ctx.obj["backend"].get_file_path(auto_file or "code.py") - print(f"Auto file: {auto_file}") auto_file_path = ctx.obj["backend"].get_auto_file_path(auto_file) print(f"Auto file path: {auto_file_path}") @@ -1296,6 +1288,7 @@ def uninstall(ctx, module): # pragma: no cover separated by a space. """ device_path = ctx.obj["DEVICE_PATH"] + print(f"Uninstalling {module} from {device_path}") for name in module: device_modules = ctx.obj["backend"].get_device_versions() name = name.lower() @@ -1309,6 +1302,8 @@ def uninstall(ctx, module): # pragma: no cover click.echo("Uninstalled '{}'.".format(name)) else: click.echo("Module '{}' not found on device.".format(name)) + continue + # pylint: disable=too-many-branches diff --git a/circup/backends.py b/circup/backends.py index 3dc75c3..9271626 100644 --- a/circup/backends.py +++ b/circup/backends.py @@ -227,34 +227,36 @@ def install_dir_http(self, source): + self.LIB_DIR_PATH + mod_name ) + target = target + "/" if target[:-1]!="/" else target url = urlparse(target) auth = HTTPBasicAuth("", url.password) print(f"target: {target}") print(f"source: {source}") # Create the top level directory. - r = requests.put((target + "/" if target[:-1]!="/" else target), auth=auth) - print(f"resp {r.content}") - r.raise_for_status() + with requests.put(target, auth=auth) as r: + print(f"resp {r.content}") + r.raise_for_status() # Traverse the directory structure and create the directories/files. for root, dirs, files in os.walk(source): rel_path = os.path.relpath(root, source) if rel_path == ".": rel_path = "" + for name in dirs: + path_to_create = urljoin( urljoin(target , rel_path + "/", allow_fragments=False) , name, allow_fragments=False) if rel_path != "" else urljoin(target , name, allow_fragments=False) + path_to_create = path_to_create + "/" if path_to_create[:-1]!="/" else path_to_create + print(f"dir_path_to_create: {path_to_create}") + with requests.put(path_to_create, auth=auth) as r: + r.raise_for_status() for name in files: with open(os.path.join(root, name), "rb") as fp: path_to_create = urljoin( urljoin(target , rel_path + "/", allow_fragments=False) , name, allow_fragments=False) if rel_path != "" else urljoin(target , name, allow_fragments=False) - # print(f"file_path_to_create: {path_to_create}") - r = requests.put( + print(f"file_path_to_create: {path_to_create}") + with requests.put( path_to_create, fp.read(), auth=auth - ) - r.raise_for_status() - for name in dirs: - path_to_create = urljoin( urljoin(target , rel_path + "/", allow_fragments=False) , name, allow_fragments=False) if rel_path != "" else urljoin(target , name, allow_fragments=False) - # print(f"dir_path_to_create: {path_to_create}") - r = requests.put(path_to_create, auth=auth) - r.raise_for_status() + ) as r: + r.raise_for_status() def get_circuitpython_version(self): """ @@ -265,16 +267,16 @@ def get_circuitpython_version(self): :return: A tuple with the version string for CircuitPython and the board ID string. """ # pylint: disable=arguments-renamed - r = requests.get(self.device_location + "/cp/version.json") - # pylint: disable=no-member - if r.status_code != requests.codes.ok: - click.secho( - f" Unable to get version from {self.device_location}: {r.status_code}", - fg="red", - ) - sys.exit(1) - # pylint: enable=no-member - ver_json = r.json() + with requests.get(self.device_location + "/cp/version.json") as r: + # pylint: disable=no-member + if r.status_code != requests.codes.ok: + click.secho( + f" Unable to get version from {self.device_location}: {r.status_code}", + fg="red", + ) + sys.exit(1) + # pylint: enable=no-member + ver_json = r.json() return ver_json.get("version"), ver_json.get("board_id") def _get_modules(self, device_lib_path): @@ -291,18 +293,18 @@ def _get_modules_http(self, url): result = {} u = urlparse(url) auth = HTTPBasicAuth("", u.password) - r = requests.get(url, auth=auth, headers={"Accept": "application/json"}) - r.raise_for_status() - - directory_mods = [] - single_file_mods = [] - for entry in r.json(): - entry_name = entry.get("name") - if entry.get("directory"): - directory_mods.append(entry_name) - else: - if entry_name.endswith(".py") or entry_name.endswith(".mpy"): - single_file_mods.append(entry_name) + with requests.get(url, auth=auth, headers={"Accept": "application/json"}) as r: + r.raise_for_status() + + directory_mods = [] + single_file_mods = [] + for entry in r.json(): + entry_name = entry.get("name") + if entry.get("directory"): + directory_mods.append(entry_name) + else: + if entry_name.endswith(".py") or entry_name.endswith(".mpy"): + single_file_mods.append(entry_name) self._get_modules_http_single_mods(auth, result, single_file_mods, url) self._get_modules_http_dir_mods(auth, directory_mods, result, url) @@ -323,34 +325,34 @@ def _get_modules_http_dir_mods(self, auth, directory_mods, result, url): dm_url = url + dm + "/" else: dm_url = dm - r = requests.get(dm_url, auth=auth, headers={"Accept": "application/json"}) - r.raise_for_status() - mpy = False - for entry in r.json(): - entry_name = entry.get("name") - if not entry.get("directory") and ( - entry_name.endswith(".py") or entry_name.endswith(".mpy") - ): - if entry_name.endswith(".mpy"): - mpy = True - r = requests.get(dm_url + entry_name, auth=auth) - r.raise_for_status() - idx = entry_name.rfind(".") - with tempfile.NamedTemporaryFile( - prefix=entry_name[:idx] + "-", - suffix=entry_name[idx:], - delete=False, - ) as fp: - fp.write(r.content) - tmp_name = fp.name - metadata = extract_metadata(tmp_name, self.logger) - os.remove(tmp_name) - if "__version__" in metadata: - metadata["path"] = dm_url - result[dm] = metadata - # break now if any of the submodules has a bad format - if metadata["__version__"] == BAD_FILE_FORMAT: - break + with requests.get(dm_url, auth=auth, headers={"Accept": "application/json"}) as r: + r.raise_for_status() + mpy = False + for entry in r.json(): + entry_name = entry.get("name") + if not entry.get("directory") and ( + entry_name.endswith(".py") or entry_name.endswith(".mpy") + ): + if entry_name.endswith(".mpy"): + mpy = True + with requests.get(dm_url + entry_name, auth=auth) as rr: + rr.raise_for_status() + idx = entry_name.rfind(".") + with tempfile.NamedTemporaryFile( + prefix=entry_name[:idx] + "-", + suffix=entry_name[idx:], + delete=False, + ) as fp: + fp.write(rr.content) + tmp_name = fp.name + metadata = extract_metadata(tmp_name, self.logger) + os.remove(tmp_name) + if "__version__" in metadata: + metadata["path"] = dm_url + result[dm] = metadata + # break now if any of the submodules has a bad format + if metadata["__version__"] == BAD_FILE_FORMAT: + break if result.get(dm) is None: result[dm] = {"path": dm_url, "mpy": mpy} @@ -367,14 +369,14 @@ def _get_modules_http_single_mods(self, auth, result, single_file_mods, url): sfm_url = url + sfm else: sfm_url = sfm - r = requests.get(sfm_url, auth=auth) - r.raise_for_status() - idx = sfm.rfind(".") - with tempfile.NamedTemporaryFile( - prefix=sfm[:idx] + "-", suffix=sfm[idx:], delete=False - ) as fp: - fp.write(r.content) - tmp_name = fp.name + with requests.get(sfm_url, auth=auth) as r: + r.raise_for_status() + idx = sfm.rfind(".") + with tempfile.NamedTemporaryFile( + prefix=sfm[:idx] + "-", suffix=sfm[idx:], delete=False + ) as fp: + fp.write(r.content) + tmp_name = fp.name metadata = extract_metadata(tmp_name, self.logger) os.remove(tmp_name) metadata["path"] = sfm_url @@ -383,8 +385,8 @@ def _get_modules_http_single_mods(self, auth, result, single_file_mods, url): def _create_library_directory(self, device_path, library_path): url = urlparse(device_path) auth = HTTPBasicAuth("", url.password) - r = requests.put(library_path, auth=auth) - r.raise_for_status() + with requests.put(library_path, auth=auth) as r: + r.raise_for_status() def _install_module_mpy(self, bundle, metadata): """ @@ -393,7 +395,6 @@ def _install_module_mpy(self, bundle, metadata): :param metadata dictionary. """ library_path = self.library_path - print(f"metadata: {metadata}") module_name = os.path.basename(metadata["path"]).replace(".py", ".mpy") if not module_name: # Must be a directory based module. @@ -438,20 +439,21 @@ def get_auto_file_path(self, auto_file_path): """ url = auto_file_path auth = HTTPBasicAuth("", self.password) - r = requests.get(url, auth=auth) - r.raise_for_status() - with open(LOCAL_CODE_PY_COPY, "w", encoding="utf-8") as f: - f.write(r.text) + with requests.get(url, auth=auth) as r: + r.raise_for_status() + with open(LOCAL_CODE_PY_COPY, "w", encoding="utf-8") as f: + f.write(r.text) return LOCAL_CODE_PY_COPY def uninstall(self, device_path, module_path): """ Uninstall given module on device using REST API. """ + print(f"Uninstalling {module_path}") url = urlparse(device_path) auth = HTTPBasicAuth("", url.password) - r = requests.delete(module_path, auth=auth) - r.raise_for_status() + with requests.delete(module_path, auth=auth) as r: + r.raise_for_status() def update(self, module): """ @@ -473,8 +475,8 @@ def _update_http(self, module): # Delete the directory (recursive) first. url = urlparse(module.path) auth = HTTPBasicAuth("", url.password) - r = requests.delete(module.path, auth=auth) - r.raise_for_status() + with requests.delete(module.path, auth=auth) as r: + r.raise_for_status() self.install_dir_http(module.bundle_path) def get_file_path(self, filename): From 7d3eb6cd78a6233b998c07c120bce82b860ca8b9 Mon Sep 17 00:00:00 2001 From: foamyguy Date: Mon, 26 Feb 2024 15:35:01 -0600 Subject: [PATCH 36/63] extract_metadata changes from #198 --- circup/shared.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/circup/shared.py b/circup/shared.py index 0b37a23..eb45430 100644 --- a/circup/shared.py +++ b/circup/shared.py @@ -62,6 +62,7 @@ def _get_modules_file(path, logger): def extract_metadata(path, logger): + # pylint: disable=too-many-locals,too-many-branches """ Given a file path, return a dictionary containing metadata extracted from dunder attributes found therein. Works with both .py and .mpy files. @@ -79,7 +80,6 @@ def extract_metadata(path, logger): :param str path: The path to the file containing the metadata. :return: The dunder based metadata found in the file, as a dictionary. """ - # pylint: disable=too-many-locals result = {} logger.info("%s", path) if path.endswith(".py"): @@ -90,7 +90,10 @@ def extract_metadata(path, logger): dunder_key_val = r"""(__\w+__)(?:\s*:\s*\w+)?\s*=\s*(?:['"]|\(\s)(.+)['"]""" for match in re.findall(dunder_key_val, content): result[match[0]] = str(match[1]) + if result: + logger.info("Extracted metadata: %s", result) elif path.endswith(".mpy"): + find_by_regexp_match = False result["mpy"] = True with open(path, "rb") as mpy_file: content = mpy_file.read() @@ -104,10 +107,20 @@ def extract_metadata(path, logger): loc = content.find(b"__version__") - 1 compatibility = (None, "7.0.0-alpha.1") elif mpy_version == b"C\x05": - # Two bytes in mpy version 5 + # Two bytes for the length of "__version__" in mpy version 5 loc = content.find(b"__version__") - 2 - compatibility = ("7.0.0-alpha.1", None) - if loc > -1: + compatibility = ("7.0.0-alpha.1", "8.99.99") + elif mpy_version == b"C\x06": + # Two bytes in mpy version 6 + find_by_regexp_match = True + compatibility = ("9.0.0-alpha.1", None) + if find_by_regexp_match: + # Too hard to find the version positionally. + # Find the first thing that looks like an x.y.z version number. + match = re.search(rb"([\d]+\.[\d]+\.[\d]+)\x00", content) + if match: + result["__version__"] = match.group(1).decode("utf-8") + elif loc > -1: # Backtrack until a byte value of the offset is reached. offset = 1 while offset < loc: @@ -128,7 +141,4 @@ def extract_metadata(path, logger): else: # not a valid MPY file result["__version__"] = BAD_FILE_FORMAT - - if result: - logger.info("Extracted metadata: %s", result) return result From 88a5f9a2e0be7329484ef2dd8ab9dddca27a4a50 Mon Sep 17 00:00:00 2001 From: foamyguy Date: Mon, 26 Feb 2024 15:54:23 -0600 Subject: [PATCH 37/63] code format --- tests/test_circup.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/test_circup.py b/tests/test_circup.py index a029395..4dce9d3 100644 --- a/tests/test_circup.py +++ b/tests/test_circup.py @@ -358,7 +358,14 @@ def test_Module_mpy_mismatch(): name, backend, repo, "1.2.3", "1.2.3", True, bundle, (None, None) ) m2 = circup.Module( - name, backend, repo, "1.2.3", "1.2.3", True, bundle, ("7.0.0-alpha.1", "8.99.99") + name, + backend, + repo, + "1.2.3", + "1.2.3", + True, + bundle, + ("7.0.0-alpha.1", "8.99.99"), ) m3 = circup.Module( name, backend, repo, "1.2.3", "1.2.3", True, bundle, (None, "7.0.0-alpha.1") From e0bbec2251453230770c3130218509982bd1e82e Mon Sep 17 00:00:00 2001 From: foamyguy Date: Mon, 26 Feb 2024 16:23:44 -0600 Subject: [PATCH 38/63] move password check and host check to WebBackend. --- circup/__init__.py | 22 +++++++++------------- circup/backends.py | 16 ++++++++++++++++ 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/circup/__init__.py b/circup/__init__.py index 8819f1a..db3d9a8 100644 --- a/circup/__init__.py +++ b/circup/__init__.py @@ -1005,7 +1005,7 @@ def libraries_from_code_py(code_py, mod_names): ) @click.pass_context def main(ctx, verbose, path, host, password, board_id, cpy_version): # pragma: no cover - # pylint: disable=too-many-arguments + # pylint: disable=too-many-arguments, too-many-branches, too-many-statements """ A tool to manage and update libraries on a CircuitPython device. """ @@ -1015,7 +1015,14 @@ def main(ctx, verbose, path, host, password, board_id, cpy_version): # pragma: using_webworkflow = "host" in ctx.params.keys() and ctx.params["host"] is not None if using_webworkflow: - ctx.obj["backend"] = WebBackend(host=host, password=password, logger=logger) + try: + ctx.obj["backend"] = WebBackend(host=host, password=password, logger=logger) + except ValueError as e: + click.secho(e, fg="red") + sys.exit(1) + except RuntimeError as e: + click.secho(e, fg="red") + sys.exit(1) else: try: ctx.obj["backend"] = DiskBackend(device_path, logger) @@ -1090,17 +1097,6 @@ def get_device_path(host, password, path): if path: device_path = path elif host: - if password is None: - click.secho("--host needs --password", fg="red") - sys.exit(1) - - # pylint: disable=no-member - # verify hostname/address - try: - socket.getaddrinfo(host, 80, proto=socket.IPPROTO_TCP) - except socket.gaierror: - click.secho("Invalid host: {}".format(host), fg="red") - sys.exit(1) # pylint: enable=no-member device_path = f"http://:{password}@" + host else: diff --git a/circup/backends.py b/circup/backends.py index 72bcd34..3a723cb 100644 --- a/circup/backends.py +++ b/circup/backends.py @@ -8,12 +8,14 @@ import os import shutil import sys +import socket import tempfile from urllib.parse import urlparse import click import requests from requests.auth import HTTPBasicAuth + from circup.shared import DATA_DIR, BAD_FILE_FORMAT, extract_metadata, _get_modules_file #: The location to store a local copy of code.py for use with --auto and @@ -185,6 +187,20 @@ class WebBackend(Backend): def __init__(self, host, password, logger): super().__init__(logger) + if password is None: + raise ValueError("--host needs --password") + + # pylint: disable=no-member + # verify hostname/address + try: + socket.getaddrinfo(host, 80, proto=socket.IPPROTO_TCP) + except socket.gaierror as exc: + raise RuntimeError( + "Invalid host: {}.".format(host) + " You should remove the 'http://'" + if "http://" in host or "https://" in host + else "" + ) from exc + self.LIB_DIR_PATH = "fs/lib/" self.host = host self.password = password From bfae6066dcad595b4e827e5a88105e5f109a3e7d Mon Sep 17 00:00:00 2001 From: foamyguy Date: Mon, 26 Feb 2024 16:30:23 -0600 Subject: [PATCH 39/63] use self.password instead of parse url --- circup/backends.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/circup/backends.py b/circup/backends.py index 3a723cb..3190a28 100644 --- a/circup/backends.py +++ b/circup/backends.py @@ -220,8 +220,8 @@ def install_file_http(self, source): + self.LIB_DIR_PATH + source.split(os.path.sep)[-1] ) - url = urlparse(target) - auth = HTTPBasicAuth("", url.password) + + auth = HTTPBasicAuth("", self.password) print(f"target: {target}") print(f"source: {source}") From dbb8374638ae092bf806051cec2d52e87c00c0f8 Mon Sep 17 00:00:00 2001 From: foamyguy Date: Mon, 26 Feb 2024 17:30:04 -0600 Subject: [PATCH 40/63] fix for json response files key. Use requests Session with retries in WebBackend. timeout on a requests call. code format --- circup/__init__.py | 21 ++++++--- circup/backends.py | 107 +++++++++++++++++++++++++++++---------------- 2 files changed, 85 insertions(+), 43 deletions(-) diff --git a/circup/__init__.py b/circup/__init__.py index 9d2cece..36d39a1 100644 --- a/circup/__init__.py +++ b/circup/__init__.py @@ -264,8 +264,12 @@ def __init__( """ self.name = name self.backend = backend - self.path = urljoin(backend.library_path,name, allow_fragments=False) if isinstance(backend,WebBackend) else os.path.join(backend.library_path, name) - url = urlparse(self.path,allow_fragments=False) + self.path = ( + urljoin(backend.library_path, name, allow_fragments=False) + if isinstance(backend, WebBackend) + else os.path.join(backend.library_path, name) + ) + url = urlparse(self.path, allow_fragments=False) if str(url.scheme).lower() in ("http", "https"): if url.path.endswith(".py") or url.path.endswith(".mpy"): self.file = os.path.basename(url.path) @@ -274,7 +278,9 @@ def __init__( ) else: self.file = None - self.name = os.path.basename(url.path if url.path[:-1]=='/' else url.path[:-1]) + self.name = os.path.basename( + url.path if url.path[:-1] == "/" else url.path[:-1] + ) else: if os.path.isfile(self.path): # Single file module. @@ -608,7 +614,9 @@ def find_modules(backend, bundles_list): if not path.endswith(os.sep) else path[:-1].split(os.sep)[-1] + os.sep ) - module_name = name if not path.find(os.sep) else module_name # should probably check for os.sep and use previous version if found + module_name = ( + name if not path.find(os.sep) else module_name + ) # should probably check for os.sep and use previous version if found m = Module( module_name, backend, @@ -1224,7 +1232,9 @@ def install(ctx, modules, pyext, requirement, auto, auto_file): # pragma: no co auto_file = "code.py" print(f"Auto file: {auto_file}") # pass a local file with "./" or "../" - is_relative = not isinstance(ctx.obj["backend"], WebBackend) or auto_file.split(os.sep)[0] in [os.path.curdir, os.path.pardir] + is_relative = not isinstance(ctx.obj["backend"], WebBackend) or auto_file.split( + os.sep + )[0] in [os.path.curdir, os.path.pardir] if not os.path.isabs(auto_file) and not is_relative: auto_file = ctx.obj["backend"].get_file_path(auto_file or "code.py") @@ -1301,7 +1311,6 @@ def uninstall(ctx, module): # pragma: no cover continue - # pylint: disable=too-many-branches diff --git a/circup/backends.py b/circup/backends.py index afd1d21..59a81b1 100644 --- a/circup/backends.py +++ b/circup/backends.py @@ -13,6 +13,7 @@ from urllib.parse import urlparse, urljoin import click import requests +from requests.adapters import HTTPAdapter from requests.auth import HTTPBasicAuth @@ -116,7 +117,11 @@ def install_module( click.echo("'{}' is already installed.".format(name)) return - library_path = os.path.join(device_path, self.LIB_DIR_PATH) if not isinstance(self,WebBackend) else urljoin(device_path,self.LIB_DIR_PATH) + library_path = ( + os.path.join(device_path, self.LIB_DIR_PATH) + if not isinstance(self, WebBackend) + else urljoin(device_path, self.LIB_DIR_PATH) + ) # Create the library directory first. self._create_library_directory(device_path, library_path) @@ -206,6 +211,8 @@ def __init__(self, host, password, logger): self.password = password self.device_location = f"http://:{self.password}@{self.host}" + self.session = requests.Session() + self.session.mount(self.device_location, HTTPAdapter(max_retries=5)) self.library_path = self.device_location + "/" + self.LIB_DIR_PATH def install_file_http(self, source): @@ -215,19 +222,14 @@ def install_file_http(self, source): """ file_name = source.split(os.path.sep) file_name = file_name[-2] if file_name[-1] == "" else file_name[-1] - target = ( - self.device_location - + "/" - + self.LIB_DIR_PATH - + file_name - ) + target = self.device_location + "/" + self.LIB_DIR_PATH + file_name auth = HTTPBasicAuth("", self.password) print(f"target: {target}") print(f"source: {source}") with open(source, "rb") as fp: - r = requests.put(target, fp.read(), auth=auth) + r = self.session.put(target, fp.read(), auth=auth) r.raise_for_status() def install_dir_http(self, source): @@ -237,20 +239,15 @@ def install_dir_http(self, source): """ mod_name = source.split(os.path.sep) mod_name = mod_name[-2] if mod_name[-1] == "" else mod_name[-1] - target = ( - self.device_location - + "/" - + self.LIB_DIR_PATH - + mod_name - ) - target = target + "/" if target[:-1]!="/" else target + target = self.device_location + "/" + self.LIB_DIR_PATH + mod_name + target = target + "/" if target[:-1] != "/" else target url = urlparse(target) auth = HTTPBasicAuth("", url.password) print(f"target: {target}") print(f"source: {source}") # Create the top level directory. - with requests.put(target, auth=auth) as r: + with self.session.put(target, auth=auth) as r: print(f"resp {r.content}") r.raise_for_status() @@ -260,18 +257,36 @@ def install_dir_http(self, source): if rel_path == ".": rel_path = "" for name in dirs: - path_to_create = urljoin( urljoin(target , rel_path + "/", allow_fragments=False) , name, allow_fragments=False) if rel_path != "" else urljoin(target , name, allow_fragments=False) - path_to_create = path_to_create + "/" if path_to_create[:-1]!="/" else path_to_create + path_to_create = ( + urljoin( + urljoin(target, rel_path + "/", allow_fragments=False), + name, + allow_fragments=False, + ) + if rel_path != "" + else urljoin(target, name, allow_fragments=False) + ) + path_to_create = ( + path_to_create + "/" + if path_to_create[:-1] != "/" + else path_to_create + ) print(f"dir_path_to_create: {path_to_create}") - with requests.put(path_to_create, auth=auth) as r: + with self.session.put(path_to_create, auth=auth) as r: r.raise_for_status() for name in files: with open(os.path.join(root, name), "rb") as fp: - path_to_create = urljoin( urljoin(target , rel_path + "/", allow_fragments=False) , name, allow_fragments=False) if rel_path != "" else urljoin(target , name, allow_fragments=False) + path_to_create = ( + urljoin( + urljoin(target, rel_path + "/", allow_fragments=False), + name, + allow_fragments=False, + ) + if rel_path != "" + else urljoin(target, name, allow_fragments=False) + ) print(f"file_path_to_create: {path_to_create}") - with requests.put( - path_to_create, fp.read(), auth=auth - ) as r: + with self.session.put(path_to_create, fp.read(), auth=auth) as r: r.raise_for_status() def get_circuitpython_version(self): @@ -283,7 +298,7 @@ def get_circuitpython_version(self): :return: A tuple with the version string for CircuitPython and the board ID string. """ # pylint: disable=arguments-renamed - with requests.get(self.device_location + "/cp/version.json") as r: + with self.session.get(self.device_location + "/cp/version.json") as r: # pylint: disable=no-member if r.status_code != requests.codes.ok: click.secho( @@ -296,6 +311,7 @@ def get_circuitpython_version(self): return ver_json.get("version"), ver_json.get("board_id") def _get_modules(self, device_lib_path): + print(f"device_lib_path {device_lib_path}") return self._get_modules_http(device_lib_path) def _get_modules_http(self, url): @@ -309,12 +325,17 @@ def _get_modules_http(self, url): result = {} u = urlparse(url) auth = HTTPBasicAuth("", u.password) - with requests.get(url, auth=auth, headers={"Accept": "application/json"}) as r: + with self.session.get( + url, auth=auth, headers={"Accept": "application/json"} + ) as r: r.raise_for_status() directory_mods = [] single_file_mods = [] - for entry in r.json(): + + for entry in r.json()["files"]: + # print(f"type: {type(entry)}") + # print(f"val: {entry}") entry_name = entry.get("name") if entry.get("directory"): directory_mods.append(entry_name) @@ -328,6 +349,7 @@ def _get_modules_http(self, url): return result def _get_modules_http_dir_mods(self, auth, directory_mods, result, url): + # pylint: disable=too-many-locals """ #TODO describe what this does @@ -337,21 +359,27 @@ def _get_modules_http_dir_mods(self, auth, directory_mods, result, url): :param url: URL of the device. """ for dm in directory_mods: - if not str(urlparse(dm).scheme).lower() in ("http", "https"): + if str(urlparse(dm).scheme).lower() not in ("http", "https"): dm_url = url + dm + "/" else: dm_url = dm - with requests.get(dm_url, auth=auth, headers={"Accept": "application/json"}) as r: + + print(f"dm_url: {dm_url}") + with self.session.get( + dm_url, auth=auth, headers={"Accept": "application/json"}, timeout=10 + ) as r: r.raise_for_status() mpy = False - for entry in r.json(): + + for entry in r.json()["files"]: entry_name = entry.get("name") if not entry.get("directory") and ( entry_name.endswith(".py") or entry_name.endswith(".mpy") ): if entry_name.endswith(".mpy"): mpy = True - with requests.get(dm_url + entry_name, auth=auth) as rr: + + with self.session.get(dm_url + entry_name, auth=auth) as rr: rr.raise_for_status() idx = entry_name.rfind(".") with tempfile.NamedTemporaryFile( @@ -381,11 +409,11 @@ def _get_modules_http_single_mods(self, auth, result, single_file_mods, url): :param url: URL of the device. """ for sfm in single_file_mods: - if not str(urlparse(sfm).scheme).lower() in ("http", "https"): + if str(urlparse(sfm).scheme).lower() not in ("http", "https"): sfm_url = url + sfm else: sfm_url = sfm - with requests.get(sfm_url, auth=auth) as r: + with self.session.get(sfm_url, auth=auth) as r: r.raise_for_status() idx = sfm.rfind(".") with tempfile.NamedTemporaryFile( @@ -401,7 +429,7 @@ def _get_modules_http_single_mods(self, auth, result, single_file_mods, url): def _create_library_directory(self, device_path, library_path): url = urlparse(device_path) auth = HTTPBasicAuth("", url.password) - with requests.put(library_path, auth=auth) as r: + with self.session.put(library_path, auth=auth) as r: r.raise_for_status() def _install_module_mpy(self, bundle, metadata): @@ -455,7 +483,7 @@ def get_auto_file_path(self, auto_file_path): """ url = auto_file_path auth = HTTPBasicAuth("", self.password) - with requests.get(url, auth=auth) as r: + with self.session.get(url, auth=auth) as r: r.raise_for_status() with open(LOCAL_CODE_PY_COPY, "w", encoding="utf-8") as f: f.write(r.text) @@ -468,7 +496,7 @@ def uninstall(self, device_path, module_path): print(f"Uninstalling {module_path}") url = urlparse(device_path) auth = HTTPBasicAuth("", url.password) - with requests.delete(module_path, auth=auth) as r: + with self.session.delete(module_path, auth=auth) as r: r.raise_for_status() def update(self, module): @@ -491,7 +519,7 @@ def _update_http(self, module): # Delete the directory (recursive) first. url = urlparse(module.path) auth = HTTPBasicAuth("", url.password) - with requests.delete(module.path, auth=auth) as r: + with self.session.delete(module.path, auth=auth) as r: r.raise_for_status() self.install_dir_http(module.bundle_path) @@ -499,7 +527,11 @@ def get_file_path(self, filename): """ retuns the full path on the device to a given file name. """ - return urljoin( urljoin(self.device_location, "fs/", allow_fragments=False), filename, allow_fragments=False) + return urljoin( + urljoin(self.device_location, "fs/", allow_fragments=False), + filename, + allow_fragments=False, + ) def is_device_present(self): """ @@ -521,6 +553,7 @@ def get_device_versions(self): """ return self.get_modules(urljoin(self.device_location, self.LIB_DIR_PATH)) + class DiskBackend(Backend): """ Backend for interacting with a device via USB Workflow From deea802a29c042326ee804e8843c3f6096388748 Mon Sep 17 00:00:00 2001 From: foamyguy Date: Sat, 2 Mar 2024 10:59:59 -0600 Subject: [PATCH 41/63] lots of prints --- circup/__init__.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/circup/__init__.py b/circup/__init__.py index 36d39a1..e0955bf 100644 --- a/circup/__init__.py +++ b/circup/__init__.py @@ -263,13 +263,18 @@ def __init__( :param (str,str) compatibility: Min and max versions of CP compatible with the mpy. """ self.name = name + print(f"name in init: {name}") self.backend = backend self.path = ( urljoin(backend.library_path, name, allow_fragments=False) if isinstance(backend, WebBackend) else os.path.join(backend.library_path, name) ) + print(f"path in init: {self.path}") url = urlparse(self.path, allow_fragments=False) + + print(url) + print(url.scheme) if str(url.scheme).lower() in ("http", "https"): if url.path.endswith(".py") or url.path.endswith(".mpy"): self.file = os.path.basename(url.path) @@ -281,15 +286,22 @@ def __init__( self.name = os.path.basename( url.path if url.path[:-1] == "/" else url.path[:-1] ) + print(f"file: {self.file}") + print(f"name: {self.name}") else: + print(f"path b4: {self.path}") if os.path.isfile(self.path): + print("isfile") # Single file module. self.file = os.path.basename(self.path) self.name = self.file.replace(".py", "").replace(".mpy", "") else: + print("directory") # Directory based module. self.file = None self.path = os.path.join(backend.library_path, name, "") + print(f"file: {self.file}") + print(f"name: {self.name}") self.repo = repo self.device_version = device_version self.bundle_version = bundle_version @@ -597,9 +609,13 @@ def find_modules(backend, bundles_list): # pylint: disable=broad-except,too-many-locals try: device_modules = backend.get_device_versions() + print(f"dev_modules: {device_modules}") bundle_modules = get_bundle_versions(bundles_list) result = [] for name, device_metadata in device_modules.items(): + print(f"name in loop: {name}") + print(f"dev_meta in loop: {device_metadata}") + if name in bundle_modules: path = device_metadata["path"] bundle_metadata = bundle_modules[name] @@ -614,9 +630,8 @@ def find_modules(backend, bundles_list): if not path.endswith(os.sep) else path[:-1].split(os.sep)[-1] + os.sep ) - module_name = ( - name if not path.find(os.sep) else module_name - ) # should probably check for os.sep and use previous version if found + print(f"module_name after 1st: {module_name}") + m = Module( module_name, backend, From ac578b852a0c08a21194c7fd82fc3c58a02df484 Mon Sep 17 00:00:00 2001 From: foamyguy Date: Sat, 2 Mar 2024 11:20:59 -0600 Subject: [PATCH 42/63] simplify logic in Module init --- circup/__init__.py | 41 ++++++++++------------------------------- 1 file changed, 10 insertions(+), 31 deletions(-) diff --git a/circup/__init__.py b/circup/__init__.py index e0955bf..6b6d0b3 100644 --- a/circup/__init__.py +++ b/circup/__init__.py @@ -275,33 +275,12 @@ def __init__( print(url) print(url.scheme) - if str(url.scheme).lower() in ("http", "https"): - if url.path.endswith(".py") or url.path.endswith(".mpy"): - self.file = os.path.basename(url.path) - self.name = ( - os.path.basename(url.path).replace(".py", "").replace(".mpy", "") - ) - else: - self.file = None - self.name = os.path.basename( - url.path if url.path[:-1] == "/" else url.path[:-1] - ) - print(f"file: {self.file}") - print(f"name: {self.name}") - else: - print(f"path b4: {self.path}") - if os.path.isfile(self.path): - print("isfile") - # Single file module. - self.file = os.path.basename(self.path) - self.name = self.file.replace(".py", "").replace(".mpy", "") - else: - print("directory") - # Directory based module. - self.file = None - self.path = os.path.join(backend.library_path, name, "") - print(f"file: {self.file}") - print(f"name: {self.name}") + + self.file = os.path.basename(url.path) + self.name = self.file.replace(".py", "").replace(".mpy", "") + print(f"file: {self.file}") + print(f"name: {self.name}") + self.repo = repo self.device_version = device_version self.bundle_version = bundle_version @@ -612,13 +591,13 @@ def find_modules(backend, bundles_list): print(f"dev_modules: {device_modules}") bundle_modules = get_bundle_versions(bundles_list) result = [] - for name, device_metadata in device_modules.items(): - print(f"name in loop: {name}") + for key, device_metadata in device_modules.items(): + print(f"key in loop: {key}") print(f"dev_meta in loop: {device_metadata}") - if name in bundle_modules: + if key in bundle_modules: path = device_metadata["path"] - bundle_metadata = bundle_modules[name] + bundle_metadata = bundle_modules[key] repo = bundle_metadata.get("__repo__") bundle = bundle_metadata.get("bundle") device_version = device_metadata.get("__version__") From 84e048e0eea373e9c36e8b60b438510de63f1813 Mon Sep 17 00:00:00 2001 From: foamyguy Date: Sat, 2 Mar 2024 11:56:25 -0600 Subject: [PATCH 43/63] fix for name when module is directory --- circup/__init__.py | 44 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/circup/__init__.py b/circup/__init__.py index 6b6d0b3..2570636 100644 --- a/circup/__init__.py +++ b/circup/__init__.py @@ -275,11 +275,51 @@ def __init__( print(url) print(url.scheme) + + print(f"url.path: {url.path}") + if url.path.endswith("/"): + self.file = None + self.name = url.path.split("/")[-2] + else: + self.file = os.path.basename(url.path) + self.name = os.path.basename(url.path).replace(".py", "").replace(".mpy", "") + + #print(f"opb: {os.path.basename(url.path)}") - self.file = os.path.basename(url.path) - self.name = self.file.replace(".py", "").replace(".mpy", "") print(f"file: {self.file}") print(f"name: {self.name}") + + # if str(url.scheme).lower() in ("http", "https"): + # if url.path.endswith(".py") or url.path.endswith(".mpy"): + # print("isfile") + # self.file = os.path.basename(url.path) + # self.name = ( + # os.path.basename(url.path).replace(".py", "").replace(".mpy", "") + # ) + # else: + # print("directory") + # self.file = None + # self.name = os.path.basename( + # url.path if url.path[:-1] == "/" else url.path[:-1] + # ) + # print(f"file: {self.file}") + # print(f"name: {self.name}") + # else: + # + # print(f"parse: {self.file.parse()}") + # print(f"path b4: {self.path}") + # if os.path.isfile(self.path): + # print("isfile") + # # Single file module. + # self.file = os.path.basename(self.path) + # self.name = self.file.replace(".py", "").replace(".mpy", "") + # else: + # print("directory") + # # Directory based module. + # self.file = None + # self.path = os.path.join(backend.library_path, name, "") + # print(f"file: {self.file}") + # print(f"name: {self.name}") self.repo = repo self.device_version = device_version From a286849393e9262ed7805099bb87c51610fbc54c Mon Sep 17 00:00:00 2001 From: foamyguy Date: Sat, 2 Mar 2024 12:06:45 -0600 Subject: [PATCH 44/63] timeouts on all requests. code format --- circup/__init__.py | 16 +++++++++------- circup/backends.py | 38 +++++++++++++++++++++++++------------- 2 files changed, 34 insertions(+), 20 deletions(-) diff --git a/circup/__init__.py b/circup/__init__.py index 2570636..2ce9f67 100644 --- a/circup/__init__.py +++ b/circup/__init__.py @@ -282,10 +282,12 @@ def __init__( self.name = url.path.split("/")[-2] else: self.file = os.path.basename(url.path) - self.name = os.path.basename(url.path).replace(".py", "").replace(".mpy", "") - - #print(f"opb: {os.path.basename(url.path)}") - + self.name = ( + os.path.basename(url.path).replace(".py", "").replace(".mpy", "") + ) + + # print(f"opb: {os.path.basename(url.path)}") + print(f"file: {self.file}") print(f"name: {self.name}") @@ -305,7 +307,7 @@ def __init__( # print(f"file: {self.file}") # print(f"name: {self.name}") # else: - # + # # print(f"parse: {self.file.parse()}") # print(f"path b4: {self.path}") # if os.path.isfile(self.path): @@ -320,7 +322,7 @@ def __init__( # self.path = os.path.join(backend.library_path, name, "") # print(f"file: {self.file}") # print(f"name: {self.name}") - + self.repo = repo self.device_version = device_version self.bundle_version = bundle_version @@ -634,7 +636,7 @@ def find_modules(backend, bundles_list): for key, device_metadata in device_modules.items(): print(f"key in loop: {key}") print(f"dev_meta in loop: {device_metadata}") - + if key in bundle_modules: path = device_metadata["path"] bundle_metadata = bundle_modules[key] diff --git a/circup/backends.py b/circup/backends.py index 59a81b1..ff3846e 100644 --- a/circup/backends.py +++ b/circup/backends.py @@ -214,6 +214,7 @@ def __init__(self, host, password, logger): self.session = requests.Session() self.session.mount(self.device_location, HTTPAdapter(max_retries=5)) self.library_path = self.device_location + "/" + self.LIB_DIR_PATH + self.timeout = 10 def install_file_http(self, source): """ @@ -229,7 +230,7 @@ def install_file_http(self, source): print(f"source: {source}") with open(source, "rb") as fp: - r = self.session.put(target, fp.read(), auth=auth) + r = self.session.put(target, fp.read(), auth=auth, timeout=self.timeout) r.raise_for_status() def install_dir_http(self, source): @@ -247,7 +248,7 @@ def install_dir_http(self, source): print(f"source: {source}") # Create the top level directory. - with self.session.put(target, auth=auth) as r: + with self.session.put(target, auth=auth, timeout=self.timeout) as r: print(f"resp {r.content}") r.raise_for_status() @@ -272,7 +273,9 @@ def install_dir_http(self, source): else path_to_create ) print(f"dir_path_to_create: {path_to_create}") - with self.session.put(path_to_create, auth=auth) as r: + with self.session.put( + path_to_create, auth=auth, timeout=self.timeout + ) as r: r.raise_for_status() for name in files: with open(os.path.join(root, name), "rb") as fp: @@ -286,7 +289,9 @@ def install_dir_http(self, source): else urljoin(target, name, allow_fragments=False) ) print(f"file_path_to_create: {path_to_create}") - with self.session.put(path_to_create, fp.read(), auth=auth) as r: + with self.session.put( + path_to_create, fp.read(), auth=auth, timeout=self.timeout + ) as r: r.raise_for_status() def get_circuitpython_version(self): @@ -298,7 +303,9 @@ def get_circuitpython_version(self): :return: A tuple with the version string for CircuitPython and the board ID string. """ # pylint: disable=arguments-renamed - with self.session.get(self.device_location + "/cp/version.json") as r: + with self.session.get( + self.device_location + "/cp/version.json", timeout=self.timeout + ) as r: # pylint: disable=no-member if r.status_code != requests.codes.ok: click.secho( @@ -326,7 +333,7 @@ def _get_modules_http(self, url): u = urlparse(url) auth = HTTPBasicAuth("", u.password) with self.session.get( - url, auth=auth, headers={"Accept": "application/json"} + url, auth=auth, headers={"Accept": "application/json"}, timeout=self.timeout ) as r: r.raise_for_status() @@ -366,7 +373,10 @@ def _get_modules_http_dir_mods(self, auth, directory_mods, result, url): print(f"dm_url: {dm_url}") with self.session.get( - dm_url, auth=auth, headers={"Accept": "application/json"}, timeout=10 + dm_url, + auth=auth, + headers={"Accept": "application/json"}, + timeout=self.timeout, ) as r: r.raise_for_status() mpy = False @@ -379,7 +389,9 @@ def _get_modules_http_dir_mods(self, auth, directory_mods, result, url): if entry_name.endswith(".mpy"): mpy = True - with self.session.get(dm_url + entry_name, auth=auth) as rr: + with self.session.get( + dm_url + entry_name, auth=auth, timeout=self.timeout + ) as rr: rr.raise_for_status() idx = entry_name.rfind(".") with tempfile.NamedTemporaryFile( @@ -413,7 +425,7 @@ def _get_modules_http_single_mods(self, auth, result, single_file_mods, url): sfm_url = url + sfm else: sfm_url = sfm - with self.session.get(sfm_url, auth=auth) as r: + with self.session.get(sfm_url, auth=auth, timeout=self.timeout) as r: r.raise_for_status() idx = sfm.rfind(".") with tempfile.NamedTemporaryFile( @@ -429,7 +441,7 @@ def _get_modules_http_single_mods(self, auth, result, single_file_mods, url): def _create_library_directory(self, device_path, library_path): url = urlparse(device_path) auth = HTTPBasicAuth("", url.password) - with self.session.put(library_path, auth=auth) as r: + with self.session.put(library_path, auth=auth, timeout=self.timeout) as r: r.raise_for_status() def _install_module_mpy(self, bundle, metadata): @@ -483,7 +495,7 @@ def get_auto_file_path(self, auto_file_path): """ url = auto_file_path auth = HTTPBasicAuth("", self.password) - with self.session.get(url, auth=auth) as r: + with self.session.get(url, auth=auth, timeout=self.timeout) as r: r.raise_for_status() with open(LOCAL_CODE_PY_COPY, "w", encoding="utf-8") as f: f.write(r.text) @@ -496,7 +508,7 @@ def uninstall(self, device_path, module_path): print(f"Uninstalling {module_path}") url = urlparse(device_path) auth = HTTPBasicAuth("", url.password) - with self.session.delete(module_path, auth=auth) as r: + with self.session.delete(module_path, auth=auth, timeout=self.timeout) as r: r.raise_for_status() def update(self, module): @@ -519,7 +531,7 @@ def _update_http(self, module): # Delete the directory (recursive) first. url = urlparse(module.path) auth = HTTPBasicAuth("", url.password) - with self.session.delete(module.path, auth=auth) as r: + with self.session.delete(module.path, auth=auth, timeout=self.timeout) as r: r.raise_for_status() self.install_dir_http(module.bundle_path) From 3ed849593d31ba6209ea2c74091376ba9b3bddb6 Mon Sep 17 00:00:00 2001 From: Tyeth Gundry Date: Sat, 2 Mar 2024 19:17:09 +0000 Subject: [PATCH 45/63] Add commandline option for timeout, sync existing REQUEST_TIMEOUT --- circup/__init__.py | 13 ++++++++++--- circup/backends.py | 4 ++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/circup/__init__.py b/circup/__init__.py index 2ce9f67..d0a3745 100644 --- a/circup/__init__.py +++ b/circup/__init__.py @@ -4,7 +4,7 @@ """ CircUp -- a utility to manage and update libraries on a CircuitPython device. """ - +import pdb import ctypes import glob @@ -646,6 +646,7 @@ def find_modules(backend, bundles_list): bundle_version = bundle_metadata.get("__version__") mpy = device_metadata["mpy"] compatibility = device_metadata.get("compatibility", (None, None)) + pdb.set_trace() module_name = ( path.split(os.sep)[-1] if not path.endswith(os.sep) @@ -1032,6 +1033,11 @@ def libraries_from_code_py(code_py, mod_names): @click.option( "--password", help="Password to use for authentication when --host is used." ) +@click.option( + "--timeout", + default=30, + help="Specify the timeout in seconds for any network operations.", +) @click.option( "--board-id", default=None, @@ -1049,19 +1055,20 @@ def libraries_from_code_py(code_py, mod_names): message="%(prog)s, A CircuitPython module updater. Version %(version)s", ) @click.pass_context -def main(ctx, verbose, path, host, password, board_id, cpy_version): # pragma: no cover +def main(ctx, verbose, path, host, password, timeout, board_id, cpy_version): # pragma: no cover # pylint: disable=too-many-arguments, too-many-branches, too-many-statements """ A tool to manage and update libraries on a CircuitPython device. """ ctx.ensure_object(dict) + ctx.obj["TIMEOUT"] = REQUESTS_TIMEOUT = timeout device_path = get_device_path(host, password, path) using_webworkflow = "host" in ctx.params.keys() and ctx.params["host"] is not None if using_webworkflow: try: - ctx.obj["backend"] = WebBackend(host=host, password=password, logger=logger) + ctx.obj["backend"] = WebBackend(host=host, password=password, logger=logger, timeout=timeout) except ValueError as e: click.secho(e, fg="red") sys.exit(1) diff --git a/circup/backends.py b/circup/backends.py index ff3846e..c392c0f 100644 --- a/circup/backends.py +++ b/circup/backends.py @@ -190,7 +190,7 @@ class WebBackend(Backend): Backend for interacting with a device via Web Workflow """ - def __init__(self, host, password, logger): + def __init__(self, host, password, logger, timeout=10): super().__init__(logger) if password is None: raise ValueError("--host needs --password") @@ -214,7 +214,7 @@ def __init__(self, host, password, logger): self.session = requests.Session() self.session.mount(self.device_location, HTTPAdapter(max_retries=5)) self.library_path = self.device_location + "/" + self.LIB_DIR_PATH - self.timeout = 10 + self.timeout = timeout def install_file_http(self, source): """ From d19e4745dfd346e42a8356d1e24c39203004ca28 Mon Sep 17 00:00:00 2001 From: tyeth Date: Sat, 2 Mar 2024 22:18:49 +0000 Subject: [PATCH 46/63] check for writable/free_space + auto_file fallback --- circup/__init__.py | 13 +++++++++---- circup/backends.py | 39 ++++++++++++++++++++++++++++++++++++--- 2 files changed, 45 insertions(+), 7 deletions(-) diff --git a/circup/__init__.py b/circup/__init__.py index d0a3745..8d568eb 100644 --- a/circup/__init__.py +++ b/circup/__init__.py @@ -4,7 +4,6 @@ """ CircUp -- a utility to manage and update libraries on a CircuitPython device. """ -import pdb import ctypes import glob @@ -646,7 +645,6 @@ def find_modules(backend, bundles_list): bundle_version = bundle_metadata.get("__version__") mpy = device_metadata["mpy"] compatibility = device_metadata.get("compatibility", (None, None)) - pdb.set_trace() module_name = ( path.split(os.sep)[-1] if not path.endswith(os.sep) @@ -1284,8 +1282,15 @@ def install(ctx, modules, pyext, requirement, auto, auto_file): # pragma: no co auto_file_path = ctx.obj["backend"].get_auto_file_path(auto_file) print(f"Auto file path: {auto_file_path}") if not os.path.isfile(auto_file_path): - click.secho(f"Auto file not found: {auto_file}", fg="red") - sys.exit(1) + # fell through to here when run from random folder on windows - ask backend. + new_auto_file = ctx.obj["backend"].get_file_path(auto_file) + if os.path.isfile(new_auto_file): + auto_file = new_auto_file + auto_file_path = ctx.obj["backend"].get_auto_file_path(auto_file) + print(f"Auto file path: {auto_file_path}") + else: + click.secho(f"Auto file not found: {auto_file}", fg="red") + sys.exit(1) requested_installs = libraries_from_code_py(auto_file_path, mod_names) else: diff --git a/circup/backends.py b/circup/backends.py index c392c0f..b85ce53 100644 --- a/circup/backends.py +++ b/circup/backends.py @@ -122,12 +122,16 @@ def install_module( if not isinstance(self, WebBackend) else urljoin(device_path, self.LIB_DIR_PATH) ) + metadata = mod_names[name] + bundle = metadata["bundle"] + bundle.size = os.path.getsize(metadata['path']) + + if self.get_free_space() < bundle.size: + self.logger.error(f"Aborted installing module {name} - not enough free space ({bundle.size} < {self.get_free_space()})") # Create the library directory first. self._create_library_directory(device_path, library_path) - metadata = mod_names[name] - bundle = metadata["bundle"] if pyext: # Use Python source for module. self._install_module_py(metadata) @@ -161,6 +165,12 @@ def get_file_path(self, filename): To be overridden by subclass """ raise NotImplementedError + + def get_free_space(self): + """ + To be overridden by subclass + """ + raise NotImplementedError def is_device_present(self): """ @@ -564,7 +574,23 @@ def get_device_versions(self): connected device. """ return self.get_modules(urljoin(self.device_location, self.LIB_DIR_PATH)) - + + def get_free_space(self): + """ + Returns the free space on the device in bytes. + """ + auth = HTTPBasicAuth("", self.password) + with self.session.get( + urljoin(self.device_location , "fs/"), auth=auth, headers={"Accept": "application/json"}, timeout=self.timeout + ) as r: + r.raise_for_status() + if r.json().get("free") is None: + self.logger.error("Unable to get free space from device.") + if r.json().get("block_size") is None: + self.logger.error("Unable to get block size from device.") + if r.json().get("writable") is None or r.json().get("writable") is False: + raise PermissionError("CircuitPython Web Workflow Device not writable\n - Remount storage as writable to device (not PC)") + return r.json()["free"] * r.json()["block_size"] # bytes class DiskBackend(Backend): """ @@ -743,3 +769,10 @@ def is_device_present(self): returns True if the device is currently connected """ return os.path.exists(self.device_location) + + def get_free_space(self): + """ + Returns the free space on the device in bytes. + """ + _, total, free = shutil.disk_usage(self.device_location) + return free From c8ff0551c5a8d23e94862386ef4fd7b1a679f3cf Mon Sep 17 00:00:00 2001 From: tyeth Date: Sat, 2 Mar 2024 23:18:05 +0000 Subject: [PATCH 47/63] Win:os.sep in Module.name/DiskBackend.library_path --- circup/__init__.py | 4 ++-- circup/backends.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/circup/__init__.py b/circup/__init__.py index 8d568eb..6d690e1 100644 --- a/circup/__init__.py +++ b/circup/__init__.py @@ -276,9 +276,9 @@ def __init__( print(url.scheme) print(f"url.path: {url.path}") - if url.path.endswith("/"): + if url.path.endswith("/") if isinstance(backend, WebBackend) else self.path.endswith(os.sep): self.file = None - self.name = url.path.split("/")[-2] + self.name = self.path.split("/" if isinstance(backend, WebBackend) else os.sep)[-2] else: self.file = os.path.basename(url.path) self.name = ( diff --git a/circup/backends.py b/circup/backends.py index b85ce53..8bfffef 100644 --- a/circup/backends.py +++ b/circup/backends.py @@ -612,7 +612,7 @@ def __init__(self, device_location, logger, boot_out=None): super().__init__(logger) self.LIB_DIR_PATH = "lib" self.device_location = device_location - self.library_path = self.device_location + "/" + self.LIB_DIR_PATH + self.library_path = os.path.join(self.device_location, os.sep + self.LIB_DIR_PATH) self.version_info = None if boot_out is not None: self.version_info = self.parse_boot_out_file(boot_out) From 5965afde9ad3f80c4b99d8de868df5d30a235d78 Mon Sep 17 00:00:00 2001 From: tyeth Date: Sun, 3 Mar 2024 01:29:46 +0000 Subject: [PATCH 48/63] Add free space check for folders + nesting --- circup/backends.py | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/circup/backends.py b/circup/backends.py index 8bfffef..4cd718c 100644 --- a/circup/backends.py +++ b/circup/backends.py @@ -125,9 +125,22 @@ def install_module( metadata = mod_names[name] bundle = metadata["bundle"] bundle.size = os.path.getsize(metadata['path']) + if os.path.isdir(metadata["path"]): + for dirpath, dirnames, filenames in os.walk(metadata["path"]): + for f in filenames: + fp = os.path.join(dirpath, f) + try: + if not os.path.islink(fp): # Ignore symbolic links + bundle.size += os.path.getsize(fp) + else: + self.logger.warning(f"Skipping symbolic link in space calculation: {fp}") + except OSError as e: + self.logger.error(f"Error: {e} - Skipping file in space calculation: {fp}") if self.get_free_space() < bundle.size: self.logger.error(f"Aborted installing module {name} - not enough free space ({bundle.size} < {self.get_free_space()})") + click.secho(f"Aborted installing module {name} - not enough free space ({bundle.size} < {self.get_free_space()})", fg="red") + return # Create the library directory first. self._create_library_directory(device_path, library_path) @@ -585,12 +598,17 @@ def get_free_space(self): ) as r: r.raise_for_status() if r.json().get("free") is None: - self.logger.error("Unable to get free space from device.") - if r.json().get("block_size") is None: + self.logger.error("Unable to get free block count from device.") + click.secho("Unable to get free block count from device.", fg="red") + elif r.json().get("block_size") is None: self.logger.error("Unable to get block size from device.") - if r.json().get("writable") is None or r.json().get("writable") is False: - raise PermissionError("CircuitPython Web Workflow Device not writable\n - Remount storage as writable to device (not PC)") - return r.json()["free"] * r.json()["block_size"] # bytes + click.secho("Unable to get block size from device.", fg="red") + elif r.json().get("writable") is None or r.json().get("writable") is False: + self.logger.error("CircuitPython Web Workflow Device not writable\n - Remount storage as writable to device (not PC)") + click.secho("CircuitPython Web Workflow Device not writable\n - Remount storage as writable to device (not PC)", fg="red") + else: + return r.json()["free"] * r.json()["block_size"] # bytes + sys.exit(1) class DiskBackend(Backend): """ From d5d8ed607d6cae8882b31fa2e8cb29c6ab035e34 Mon Sep 17 00:00:00 2001 From: tyeth Date: Sun, 3 Mar 2024 02:03:23 +0000 Subject: [PATCH 49/63] Set logging to truncate log file automatically --- circup/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/circup/__init__.py b/circup/__init__.py index 6d690e1..70079c8 100644 --- a/circup/__init__.py +++ b/circup/__init__.py @@ -9,6 +9,7 @@ import glob import json import logging +from logging.handlers import RotatingFileHandler import os import re import shutil @@ -83,7 +84,7 @@ # Setup logging. logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) -logfile_handler = logging.FileHandler(LOGFILE) +logfile_handler = RotatingFileHandler(LOGFILE, maxBytes=10_000_000, backupCount=0) log_formatter = logging.Formatter( "%(asctime)s %(levelname)s: %(message)s", datefmt="%m/%d/%Y %H:%M:%S" ) From 1f8443f164eb048d7d65618fe36eac3dcdbb4d3c Mon Sep 17 00:00:00 2001 From: foamyguy Date: Sun, 3 Mar 2024 09:45:51 -0600 Subject: [PATCH 50/63] remove extra sep --- circup/backends.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/circup/backends.py b/circup/backends.py index 4cd718c..453f545 100644 --- a/circup/backends.py +++ b/circup/backends.py @@ -630,7 +630,7 @@ def __init__(self, device_location, logger, boot_out=None): super().__init__(logger) self.LIB_DIR_PATH = "lib" self.device_location = device_location - self.library_path = os.path.join(self.device_location, os.sep + self.LIB_DIR_PATH) + self.library_path = os.path.join(self.device_location, self.LIB_DIR_PATH) self.version_info = None if boot_out is not None: self.version_info = self.parse_boot_out_file(boot_out) @@ -706,8 +706,11 @@ def _install_module_mpy(self, bundle, metadata): # Copy the directory. shutil.copytree(bundle_path, target_path) elif os.path.isfile(bundle_path): + target = os.path.basename(bundle_path) + print(f"lib path: {self.library_path} target: {target}") target_path = os.path.join(self.library_path, target) + print(f"target path: {target_path}") # Copy file. shutil.copyfile(bundle_path, target_path) else: From 42d9fe6c08212d3587c4ebbb36331d3838792577 Mon Sep 17 00:00:00 2001 From: foamyguy Date: Sun, 3 Mar 2024 11:52:10 -0600 Subject: [PATCH 51/63] code format and pylint --- circup/__init__.py | 21 ++++++++++++++----- circup/backends.py | 50 ++++++++++++++++++++++++++++++++++------------ 2 files changed, 53 insertions(+), 18 deletions(-) diff --git a/circup/__init__.py b/circup/__init__.py index 70079c8..4f09b3a 100644 --- a/circup/__init__.py +++ b/circup/__init__.py @@ -277,9 +277,15 @@ def __init__( print(url.scheme) print(f"url.path: {url.path}") - if url.path.endswith("/") if isinstance(backend, WebBackend) else self.path.endswith(os.sep): + if ( + url.path.endswith("/") + if isinstance(backend, WebBackend) + else self.path.endswith(os.sep) + ): self.file = None - self.name = self.path.split("/" if isinstance(backend, WebBackend) else os.sep)[-2] + self.name = self.path.split( + "/" if isinstance(backend, WebBackend) else os.sep + )[-2] else: self.file = os.path.basename(url.path) self.name = ( @@ -1054,12 +1060,15 @@ def libraries_from_code_py(code_py, mod_names): message="%(prog)s, A CircuitPython module updater. Version %(version)s", ) @click.pass_context -def main(ctx, verbose, path, host, password, timeout, board_id, cpy_version): # pragma: no cover - # pylint: disable=too-many-arguments, too-many-branches, too-many-statements +def main( + ctx, verbose, path, host, password, timeout, board_id, cpy_version +): # pragma: no cover + # pylint: disable=too-many-arguments, too-many-branches, too-many-statements, too-many-locals """ A tool to manage and update libraries on a CircuitPython device. """ ctx.ensure_object(dict) + global REQUESTS_TIMEOUT ctx.obj["TIMEOUT"] = REQUESTS_TIMEOUT = timeout device_path = get_device_path(host, password, path) @@ -1067,7 +1076,9 @@ def main(ctx, verbose, path, host, password, timeout, board_id, cpy_version): # if using_webworkflow: try: - ctx.obj["backend"] = WebBackend(host=host, password=password, logger=logger, timeout=timeout) + ctx.obj["backend"] = WebBackend( + host=host, password=password, logger=logger, timeout=timeout + ) except ValueError as e: click.secho(e, fg="red") sys.exit(1) diff --git a/circup/backends.py b/circup/backends.py index 453f545..0bacf3e 100644 --- a/circup/backends.py +++ b/circup/backends.py @@ -91,7 +91,7 @@ def _install_module_mpy(self, bundle, metadata): """ raise NotImplementedError - # pylint: disable=too-many-locals,too-many-branches,too-many-arguments + # pylint: disable=too-many-locals,too-many-branches,too-many-arguments,too-many-nested-blocks def install_module( self, device_path, device_modules, name, pyext, mod_names ): # pragma: no cover @@ -124,8 +124,9 @@ def install_module( ) metadata = mod_names[name] bundle = metadata["bundle"] - bundle.size = os.path.getsize(metadata['path']) + bundle.size = os.path.getsize(metadata["path"]) if os.path.isdir(metadata["path"]): + # pylint: disable=unused-variable for dirpath, dirnames, filenames in os.walk(metadata["path"]): for f in filenames: fp = os.path.join(dirpath, f) @@ -133,13 +134,24 @@ def install_module( if not os.path.islink(fp): # Ignore symbolic links bundle.size += os.path.getsize(fp) else: - self.logger.warning(f"Skipping symbolic link in space calculation: {fp}") + self.logger.warning( + f"Skipping symbolic link in space calculation: {fp}" + ) except OSError as e: - self.logger.error(f"Error: {e} - Skipping file in space calculation: {fp}") + self.logger.error( + f"Error: {e} - Skipping file in space calculation: {fp}" + ) if self.get_free_space() < bundle.size: - self.logger.error(f"Aborted installing module {name} - not enough free space ({bundle.size} < {self.get_free_space()})") - click.secho(f"Aborted installing module {name} - not enough free space ({bundle.size} < {self.get_free_space()})", fg="red") + self.logger.error( + f"Aborted installing module {name} - " + f"not enough free space ({bundle.size} < {self.get_free_space()})" + ) + click.secho( + f"Aborted installing module {name} - " + f"not enough free space ({bundle.size} < {self.get_free_space()})", + fg="red", + ) return # Create the library directory first. @@ -178,7 +190,7 @@ def get_file_path(self, filename): To be overridden by subclass """ raise NotImplementedError - + def get_free_space(self): """ To be overridden by subclass @@ -587,14 +599,17 @@ def get_device_versions(self): connected device. """ return self.get_modules(urljoin(self.device_location, self.LIB_DIR_PATH)) - + def get_free_space(self): """ Returns the free space on the device in bytes. """ auth = HTTPBasicAuth("", self.password) with self.session.get( - urljoin(self.device_location , "fs/"), auth=auth, headers={"Accept": "application/json"}, timeout=self.timeout + urljoin(self.device_location, "fs/"), + auth=auth, + headers={"Accept": "application/json"}, + timeout=self.timeout, ) as r: r.raise_for_status() if r.json().get("free") is None: @@ -604,12 +619,20 @@ def get_free_space(self): self.logger.error("Unable to get block size from device.") click.secho("Unable to get block size from device.", fg="red") elif r.json().get("writable") is None or r.json().get("writable") is False: - self.logger.error("CircuitPython Web Workflow Device not writable\n - Remount storage as writable to device (not PC)") - click.secho("CircuitPython Web Workflow Device not writable\n - Remount storage as writable to device (not PC)", fg="red") + self.logger.error( + "CircuitPython Web Workflow Device not writable\n - " + "Remount storage as writable to device (not PC)" + ) + click.secho( + "CircuitPython Web Workflow Device not writable\n - " + "Remount storage as writable to device (not PC)", + fg="red", + ) else: - return r.json()["free"] * r.json()["block_size"] # bytes + return r.json()["free"] * r.json()["block_size"] # bytes sys.exit(1) + class DiskBackend(Backend): """ Backend for interacting with a device via USB Workflow @@ -706,7 +729,7 @@ def _install_module_mpy(self, bundle, metadata): # Copy the directory. shutil.copytree(bundle_path, target_path) elif os.path.isfile(bundle_path): - + target = os.path.basename(bundle_path) print(f"lib path: {self.library_path} target: {target}") target_path = os.path.join(self.library_path, target) @@ -795,5 +818,6 @@ def get_free_space(self): """ Returns the free space on the device in bytes. """ + # pylint: disable=unused-variable _, total, free = shutil.disk_usage(self.device_location) return free From 1c8b32c3268ed5cebcf29543f26f1051e6fda7a0 Mon Sep 17 00:00:00 2001 From: foamyguy Date: Sun, 3 Mar 2024 11:57:04 -0600 Subject: [PATCH 52/63] ignore locals --- circup/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circup/__init__.py b/circup/__init__.py index 4f09b3a..b8905ce 100644 --- a/circup/__init__.py +++ b/circup/__init__.py @@ -95,7 +95,7 @@ __version__ = "0.0.0-auto.0" __repo__ = "https://github.com/adafruit/circup.git" - +# pylint: disable=too-many-locals class Bundle: """ All the links and file names for a bundle From 3ec6e34bd3811a22cb2e2ecf1d6286a6be3eb935 Mon Sep 17 00:00:00 2001 From: foamyguy Date: Sun, 3 Mar 2024 12:14:31 -0600 Subject: [PATCH 53/63] try moving this? --- circup/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/circup/__init__.py b/circup/__init__.py index b8905ce..f372a91 100644 --- a/circup/__init__.py +++ b/circup/__init__.py @@ -95,7 +95,7 @@ __version__ = "0.0.0-auto.0" __repo__ = "https://github.com/adafruit/circup.git" -# pylint: disable=too-many-locals + class Bundle: """ All the links and file names for a bundle @@ -1063,10 +1063,10 @@ def libraries_from_code_py(code_py, mod_names): def main( ctx, verbose, path, host, password, timeout, board_id, cpy_version ): # pragma: no cover - # pylint: disable=too-many-arguments, too-many-branches, too-many-statements, too-many-locals """ A tool to manage and update libraries on a CircuitPython device. """ + # pylint: disable=too-many-arguments,too-many-branches,too-many-statements,too-many-locals ctx.ensure_object(dict) global REQUESTS_TIMEOUT ctx.obj["TIMEOUT"] = REQUESTS_TIMEOUT = timeout From 08e0c9455d9961ad2907bfbffac17a8d0bd8516d Mon Sep 17 00:00:00 2001 From: foamyguy Date: Sun, 3 Mar 2024 12:17:02 -0600 Subject: [PATCH 54/63] add a different one? --- circup/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/circup/__init__.py b/circup/__init__.py index f372a91..42f001a 100644 --- a/circup/__init__.py +++ b/circup/__init__.py @@ -1059,6 +1059,7 @@ def libraries_from_code_py(code_py, mod_names): prog_name="CircUp", message="%(prog)s, A CircuitPython module updater. Version %(version)s", ) +# pylint: disable=too-many-locals @click.pass_context def main( ctx, verbose, path, host, password, timeout, board_id, cpy_version From 2c8d3e93e9b798e3ab4dfcb502f6aeda3d82fc08 Mon Sep 17 00:00:00 2001 From: foamyguy Date: Sun, 3 Mar 2024 12:18:56 -0600 Subject: [PATCH 55/63] different line --- circup/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circup/__init__.py b/circup/__init__.py index 42f001a..701e20b 100644 --- a/circup/__init__.py +++ b/circup/__init__.py @@ -1059,7 +1059,6 @@ def libraries_from_code_py(code_py, mod_names): prog_name="CircUp", message="%(prog)s, A CircuitPython module updater. Version %(version)s", ) -# pylint: disable=too-many-locals @click.pass_context def main( ctx, verbose, path, host, password, timeout, board_id, cpy_version @@ -1068,6 +1067,7 @@ def main( A tool to manage and update libraries on a CircuitPython device. """ # pylint: disable=too-many-arguments,too-many-branches,too-many-statements,too-many-locals + # pylint: disable=too-many-locals ctx.ensure_object(dict) global REQUESTS_TIMEOUT ctx.obj["TIMEOUT"] = REQUESTS_TIMEOUT = timeout From 3622c3df67697156b1ea9dbb5033871c9e171ec8 Mon Sep 17 00:00:00 2001 From: foamyguy Date: Sun, 3 Mar 2024 12:22:16 -0600 Subject: [PATCH 56/63] inside the arguments --- circup/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/circup/__init__.py b/circup/__init__.py index 701e20b..6fdcb5b 100644 --- a/circup/__init__.py +++ b/circup/__init__.py @@ -1060,14 +1060,13 @@ def libraries_from_code_py(code_py, mod_names): message="%(prog)s, A CircuitPython module updater. Version %(version)s", ) @click.pass_context -def main( +def main( # pylint: disable=too-many-locals ctx, verbose, path, host, password, timeout, board_id, cpy_version ): # pragma: no cover """ A tool to manage and update libraries on a CircuitPython device. """ # pylint: disable=too-many-arguments,too-many-branches,too-many-statements,too-many-locals - # pylint: disable=too-many-locals ctx.ensure_object(dict) global REQUESTS_TIMEOUT ctx.obj["TIMEOUT"] = REQUESTS_TIMEOUT = timeout From 3cd60f12a9993e084d65a46942a7a171e73f9970 Mon Sep 17 00:00:00 2001 From: foamyguy Date: Sun, 3 Mar 2024 12:58:38 -0600 Subject: [PATCH 57/63] fix directory_module test --- tests/test_circup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_circup.py b/tests/test_circup.py index 4dce9d3..73672cb 100644 --- a/tests/test_circup.py +++ b/tests/test_circup.py @@ -249,7 +249,7 @@ def test_Module_init_directory_module(): Ensure the Module instance is set up as expected and logged, as if for a directory based Python module. """ - name = "dir_module" + name = "dir_module/" path = os.path.join("mock_device", "lib", f"{name}", "") repo = "https://github.com/adafruit/SomeLibrary.git" device_version = "1.2.3" From bc30d36255e553a9c7b40bb425d9d2e3b6c715da Mon Sep 17 00:00:00 2001 From: foamyguy Date: Sun, 3 Mar 2024 16:01:57 -0600 Subject: [PATCH 58/63] lookup host from circuitpyhon.local --- circup/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/circup/__init__.py b/circup/__init__.py index 6fdcb5b..4f265c5 100644 --- a/circup/__init__.py +++ b/circup/__init__.py @@ -1075,6 +1075,12 @@ def main( # pylint: disable=too-many-locals using_webworkflow = "host" in ctx.params.keys() and ctx.params["host"] is not None if using_webworkflow: + if host == "circuitpython.local": + click.echo("Checking versions.json on circuitpython.local to find hostname") + versions_resp = requests.get("http://circuitpython.local/cp/version.json", timeout=timeout) + host = f'{versions_resp.json()["hostname"]}.local' + click.echo(f"Using hostname: {host}") + device_path = device_path.replace("circuitpython.local", host) try: ctx.obj["backend"] = WebBackend( host=host, password=password, logger=logger, timeout=timeout From 73d5872ec74db8a43b574915669370b5e8ea8563 Mon Sep 17 00:00:00 2001 From: foamyguy Date: Sun, 3 Mar 2024 16:02:34 -0600 Subject: [PATCH 59/63] code format --- circup/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/circup/__init__.py b/circup/__init__.py index 4f265c5..c6ad872 100644 --- a/circup/__init__.py +++ b/circup/__init__.py @@ -1077,7 +1077,9 @@ def main( # pylint: disable=too-many-locals if using_webworkflow: if host == "circuitpython.local": click.echo("Checking versions.json on circuitpython.local to find hostname") - versions_resp = requests.get("http://circuitpython.local/cp/version.json", timeout=timeout) + versions_resp = requests.get( + "http://circuitpython.local/cp/version.json", timeout=timeout + ) host = f'{versions_resp.json()["hostname"]}.local' click.echo(f"Using hostname: {host}") device_path = device_path.replace("circuitpython.local", host) From 0c5dec02e6be9a795ac31d7eaf550bff3142d134 Mon Sep 17 00:00:00 2001 From: foamyguy Date: Sun, 3 Mar 2024 16:17:46 -0600 Subject: [PATCH 60/63] use free_space instead of device_versions to verify presence --- circup/backends.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circup/backends.py b/circup/backends.py index 0bacf3e..f6dfa8f 100644 --- a/circup/backends.py +++ b/circup/backends.py @@ -585,7 +585,7 @@ def is_device_present(self): returns True if the device is currently connected """ try: - _ = self.get_device_versions() + _ = self.get_free_space() return True except requests.exceptions.ConnectionError: return False From 258f08494f9e647071c9f46ef6e8e128611dc912 Mon Sep 17 00:00:00 2001 From: foamyguy Date: Sun, 3 Mar 2024 17:08:54 -0600 Subject: [PATCH 61/63] cleanup and add error message --- circup/__init__.py | 50 +++------------------------------------------- circup/backends.py | 30 ++++++---------------------- 2 files changed, 9 insertions(+), 71 deletions(-) diff --git a/circup/__init__.py b/circup/__init__.py index c6ad872..12fc2d3 100644 --- a/circup/__init__.py +++ b/circup/__init__.py @@ -9,6 +9,7 @@ import glob import json import logging +import time from logging.handlers import RotatingFileHandler import os import re @@ -263,20 +264,15 @@ def __init__( :param (str,str) compatibility: Min and max versions of CP compatible with the mpy. """ self.name = name - print(f"name in init: {name}") self.backend = backend self.path = ( urljoin(backend.library_path, name, allow_fragments=False) if isinstance(backend, WebBackend) else os.path.join(backend.library_path, name) ) - print(f"path in init: {self.path}") - url = urlparse(self.path, allow_fragments=False) - print(url) - print(url.scheme) + url = urlparse(self.path, allow_fragments=False) - print(f"url.path: {url.path}") if ( url.path.endswith("/") if isinstance(backend, WebBackend) @@ -292,43 +288,6 @@ def __init__( os.path.basename(url.path).replace(".py", "").replace(".mpy", "") ) - # print(f"opb: {os.path.basename(url.path)}") - - print(f"file: {self.file}") - print(f"name: {self.name}") - - # if str(url.scheme).lower() in ("http", "https"): - # if url.path.endswith(".py") or url.path.endswith(".mpy"): - # print("isfile") - # self.file = os.path.basename(url.path) - # self.name = ( - # os.path.basename(url.path).replace(".py", "").replace(".mpy", "") - # ) - # else: - # print("directory") - # self.file = None - # self.name = os.path.basename( - # url.path if url.path[:-1] == "/" else url.path[:-1] - # ) - # print(f"file: {self.file}") - # print(f"name: {self.name}") - # else: - # - # print(f"parse: {self.file.parse()}") - # print(f"path b4: {self.path}") - # if os.path.isfile(self.path): - # print("isfile") - # # Single file module. - # self.file = os.path.basename(self.path) - # self.name = self.file.replace(".py", "").replace(".mpy", "") - # else: - # print("directory") - # # Directory based module. - # self.file = None - # self.path = os.path.join(backend.library_path, name, "") - # print(f"file: {self.file}") - # print(f"name: {self.name}") - self.repo = repo self.device_version = device_version self.bundle_version = bundle_version @@ -636,12 +595,9 @@ def find_modules(backend, bundles_list): # pylint: disable=broad-except,too-many-locals try: device_modules = backend.get_device_versions() - print(f"dev_modules: {device_modules}") bundle_modules = get_bundle_versions(bundles_list) result = [] for key, device_metadata in device_modules.items(): - print(f"key in loop: {key}") - print(f"dev_meta in loop: {device_metadata}") if key in bundle_modules: path = device_metadata["path"] @@ -657,7 +613,6 @@ def find_modules(backend, bundles_list): if not path.endswith(os.sep) else path[:-1].split(os.sep)[-1] + os.sep ) - print(f"module_name after 1st: {module_name}") m = Module( module_name, @@ -1089,6 +1044,7 @@ def main( # pylint: disable=too-many-locals ) except ValueError as e: click.secho(e, fg="red") + time.sleep(0.3) sys.exit(1) except RuntimeError as e: click.secho(e, fg="red") diff --git a/circup/backends.py b/circup/backends.py index f6dfa8f..8e34df8 100644 --- a/circup/backends.py +++ b/circup/backends.py @@ -238,7 +238,7 @@ def __init__(self, host, password, logger, timeout=10): raise RuntimeError( "Invalid host: {}.".format(host) + " You should remove the 'http://'" if "http://" in host or "https://" in host - else "" + else "Could not find or connect to specified device" ) from exc self.LIB_DIR_PATH = "fs/lib/" @@ -261,8 +261,6 @@ def install_file_http(self, source): target = self.device_location + "/" + self.LIB_DIR_PATH + file_name auth = HTTPBasicAuth("", self.password) - print(f"target: {target}") - print(f"source: {source}") with open(source, "rb") as fp: r = self.session.put(target, fp.read(), auth=auth, timeout=self.timeout) @@ -279,12 +277,9 @@ def install_dir_http(self, source): target = target + "/" if target[:-1] != "/" else target url = urlparse(target) auth = HTTPBasicAuth("", url.password) - print(f"target: {target}") - print(f"source: {source}") # Create the top level directory. with self.session.put(target, auth=auth, timeout=self.timeout) as r: - print(f"resp {r.content}") r.raise_for_status() # Traverse the directory structure and create the directories/files. @@ -307,7 +302,7 @@ def install_dir_http(self, source): if path_to_create[:-1] != "/" else path_to_create ) - print(f"dir_path_to_create: {path_to_create}") + with self.session.put( path_to_create, auth=auth, timeout=self.timeout ) as r: @@ -323,7 +318,6 @@ def install_dir_http(self, source): if rel_path != "" else urljoin(target, name, allow_fragments=False) ) - print(f"file_path_to_create: {path_to_create}") with self.session.put( path_to_create, fp.read(), auth=auth, timeout=self.timeout ) as r: @@ -353,7 +347,6 @@ def get_circuitpython_version(self): return ver_json.get("version"), ver_json.get("board_id") def _get_modules(self, device_lib_path): - print(f"device_lib_path {device_lib_path}") return self._get_modules_http(device_lib_path) def _get_modules_http(self, url): @@ -376,8 +369,7 @@ def _get_modules_http(self, url): single_file_mods = [] for entry in r.json()["files"]: - # print(f"type: {type(entry)}") - # print(f"val: {entry}") + entry_name = entry.get("name") if entry.get("directory"): directory_mods.append(entry_name) @@ -406,7 +398,6 @@ def _get_modules_http_dir_mods(self, auth, directory_mods, result, url): else: dm_url = dm - print(f"dm_url: {dm_url}") with self.session.get( dm_url, auth=auth, @@ -485,7 +476,6 @@ def _install_module_mpy(self, bundle, metadata): :param library_path library path :param metadata dictionary. """ - library_path = self.library_path module_name = os.path.basename(metadata["path"]).replace(".py", ".mpy") if not module_name: # Must be a directory based module. @@ -495,15 +485,9 @@ def _install_module_mpy(self, bundle, metadata): bundle_path = os.path.join(bundle.lib_dir(bundle_platform), module_name) if os.path.isdir(bundle_path): - print(f"456 library_path: {library_path}") - print(f"456 module_name: {module_name}") - self.install_dir_http(bundle_path) elif os.path.isfile(bundle_path): - target = os.path.basename(bundle_path) - print(f"123 library_path: {library_path}") - print(f"123 target: {target}") self.install_file_http(bundle_path) else: @@ -540,7 +524,6 @@ def uninstall(self, device_path, module_path): """ Uninstall given module on device using REST API. """ - print(f"Uninstalling {module_path}") url = urlparse(device_path) auth = HTTPBasicAuth("", url.password) with self.session.delete(module_path, auth=auth, timeout=self.timeout) as r: @@ -585,7 +568,7 @@ def is_device_present(self): returns True if the device is currently connected """ try: - _ = self.get_free_space() + _ = self.session.get(f"{self.device_location}/cp/version.json") return True except requests.exceptions.ConnectionError: return False @@ -691,7 +674,6 @@ def get_circuitpython_version(self): ) self.logger.error("boot_out.txt not found.") sys.exit(1) - print((circuit_python, board_id)) return circuit_python, board_id return self.version_info @@ -731,9 +713,9 @@ def _install_module_mpy(self, bundle, metadata): elif os.path.isfile(bundle_path): target = os.path.basename(bundle_path) - print(f"lib path: {self.library_path} target: {target}") + target_path = os.path.join(self.library_path, target) - print(f"target path: {target_path}") + # Copy file. shutil.copyfile(bundle_path, target_path) else: From 1ef7651d6f20a0ab417db34608c2afeff0cfabe7 Mon Sep 17 00:00:00 2001 From: foamyguy Date: Sun, 3 Mar 2024 17:14:21 -0600 Subject: [PATCH 62/63] function doc --- circup/backends.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/circup/backends.py b/circup/backends.py index 8e34df8..780bdc5 100644 --- a/circup/backends.py +++ b/circup/backends.py @@ -385,7 +385,8 @@ def _get_modules_http(self, url): def _get_modules_http_dir_mods(self, auth, directory_mods, result, url): # pylint: disable=too-many-locals """ - #TODO describe what this does + Builds result dictionary with keys containing module names and values containing a + dictionary with metadata bout the module like version, compatibility, mpy or not etc. :param auth HTTP authentication. :param directory_mods list of modules. From 5479ff44a662e634d16bc12956d4d0a12531736c Mon Sep 17 00:00:00 2001 From: foamyguy Date: Sun, 3 Mar 2024 18:50:22 -0600 Subject: [PATCH 63/63] non_writeable drive error message. --- circup/backends.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/circup/backends.py b/circup/backends.py index 780bdc5..754cbac 100644 --- a/circup/backends.py +++ b/circup/backends.py @@ -220,6 +220,15 @@ def parse_boot_out_file(boot_out_contents): return circuit_python, board_id +def _writeable_error(): + click.secho( + "CircuitPython Web Workflow Device not writable\n - " + "Remount storage as writable to device (not PC)", + fg="red", + ) + sys.exit(1) + + class WebBackend(Backend): """ Backend for interacting with a device via Web Workflow @@ -264,6 +273,8 @@ def install_file_http(self, source): with open(source, "rb") as fp: r = self.session.put(target, fp.read(), auth=auth, timeout=self.timeout) + if r.status_code == 409: + _writeable_error() r.raise_for_status() def install_dir_http(self, source): @@ -280,6 +291,8 @@ def install_dir_http(self, source): # Create the top level directory. with self.session.put(target, auth=auth, timeout=self.timeout) as r: + if r.status_code == 409: + _writeable_error() r.raise_for_status() # Traverse the directory structure and create the directories/files. @@ -306,6 +319,8 @@ def install_dir_http(self, source): with self.session.put( path_to_create, auth=auth, timeout=self.timeout ) as r: + if r.status_code == 409: + _writeable_error() r.raise_for_status() for name in files: with open(os.path.join(root, name), "rb") as fp: @@ -321,6 +336,8 @@ def install_dir_http(self, source): with self.session.put( path_to_create, fp.read(), auth=auth, timeout=self.timeout ) as r: + if r.status_code == 409: + _writeable_error() r.raise_for_status() def get_circuitpython_version(self): @@ -469,6 +486,8 @@ def _create_library_directory(self, device_path, library_path): url = urlparse(device_path) auth = HTTPBasicAuth("", url.password) with self.session.put(library_path, auth=auth, timeout=self.timeout) as r: + if r.status_code == 409: + _writeable_error() r.raise_for_status() def _install_module_mpy(self, bundle, metadata): @@ -528,6 +547,8 @@ def uninstall(self, device_path, module_path): url = urlparse(device_path) auth = HTTPBasicAuth("", url.password) with self.session.delete(module_path, auth=auth, timeout=self.timeout) as r: + if r.status_code == 409: + _writeable_error() r.raise_for_status() def update(self, module): @@ -551,6 +572,8 @@ def _update_http(self, module): url = urlparse(module.path) auth = HTTPBasicAuth("", url.password) with self.session.delete(module.path, auth=auth, timeout=self.timeout) as r: + if r.status_code == 409: + _writeable_error() r.raise_for_status() self.install_dir_http(module.bundle_path)