From 04836d5725eba690a80033e66aa43e372bbaa2b1 Mon Sep 17 00:00:00 2001 From: foamyguy Date: Fri, 26 Apr 2024 16:57:40 -0500 Subject: [PATCH 1/4] local module installs --- circup/backends.py | 71 +++++++++++++++++++++++++++++------------ circup/command_utils.py | 33 +++++++++++-------- 2 files changed, 70 insertions(+), 34 deletions(-) diff --git a/circup/backends.py b/circup/backends.py index 56758f9..310e877 100644 --- a/circup/backends.py +++ b/circup/backends.py @@ -89,7 +89,14 @@ def install_module_mpy(self, bundle, metadata): To be overridden by subclass """ raise NotImplementedError - + + def copy_file(self, target_file, location_to_paste): + """Paste a copy of the specified file at the location given""" + """ + To be overridden by subclass + """ + raise NotImplementedError + # pylint: disable=too-many-locals,too-many-branches,too-many-arguments,too-many-nested-blocks def install_module( self, device_path, device_modules, name, pyext, mod_names, upgrade=False @@ -109,9 +116,16 @@ def install_module( with get_bundle_versions() :param bool upgrade: Upgrade the specified modules if they're already installed. """ + local_path = None if not name: click.echo("No module name(s) provided.") - elif name in mod_names: + elif name in mod_names or os.path.exists(name): + if os.path.exists(name): + # local file exists use that. + local_path = name + name = local_path.split(os.path.sep)[-1] + name = name.replace(".py", "").replace(".mpy", "") + # Grab device modules to check if module already installed if name in device_modules: if not upgrade: @@ -128,15 +142,21 @@ def install_module( _metadata = _mod_names[name] module_path = _metadata["path"] self.uninstall(device_path, module_path) - - library_path = ( - os.path.join(device_path, self.LIB_DIR_PATH) - if not isinstance(self, WebBackend) - else urljoin(device_path, self.LIB_DIR_PATH) - ) - metadata = mod_names[name] - bundle = metadata["bundle"] - bundle.size = os.path.getsize(metadata["path"]) + + new_module_size = 0 + if local_path is None: + library_path = ( + os.path.join(device_path, self.LIB_DIR_PATH) + if not isinstance(self, WebBackend) + else urljoin(device_path, self.LIB_DIR_PATH) + ) + metadata = mod_names[name] + bundle = metadata["bundle"] + else: + library_path = local_path + metadata = {"path": local_path} + + new_module_size = os.path.getsize(metadata["path"]) if os.path.isdir(metadata["path"]): # pylint: disable=unused-variable for dirpath, dirnames, filenames in os.walk(metadata["path"]): @@ -144,7 +164,7 @@ def install_module( fp = os.path.join(dirpath, f) try: if not os.path.islink(fp): # Ignore symbolic links - bundle.size += os.path.getsize(fp) + new_module_size += os.path.getsize(fp) else: self.logger.warning( f"Skipping symbolic link in space calculation: {fp}" @@ -154,27 +174,29 @@ def install_module( f"Error: {e} - Skipping file in space calculation: {fp}" ) - if self.get_free_space() < bundle.size: + if self.get_free_space() < new_module_size: self.logger.error( f"Aborted installing module {name} - " - f"not enough free space ({bundle.size} < {self.get_free_space()})" + f"not enough free space ({new_module_size} < {self.get_free_space()})" ) click.secho( f"Aborted installing module {name} - " - f"not enough free space ({bundle.size} < {self.get_free_space()})", + f"not enough free space ({new_module_size} < {self.get_free_space()})", fg="red", ) return # Create the library directory first. self._create_library_directory(device_path, library_path) - - if pyext: - # Use Python source for module. - self.install_module_py(metadata) + if local_path is None: + if pyext: + # Use Python source for module. + self.install_module_py(metadata) + else: + # Use pre-compiled mpy modules. + self.install_module_mpy(bundle, metadata) else: - # Use pre-compiled mpy modules. - self.install_module_mpy(bundle, metadata) + self.copy_file(library_path, "lib") click.echo("Installed '{}'.".format(name)) else: click.echo("Unknown module named, '{}'.".format(name)) @@ -775,6 +797,13 @@ def _create_library_directory(self, device_path, library_path): if not os.path.exists(library_path): # pragma: no cover os.makedirs(library_path) + def copy_file(self, target_file, location_to_paste): + target_filename = target_file.split(os.path.sep)[-1] + if os.path.isdir(target_file): + shutil.copytree(target_file, os.path.join(self.device_location, location_to_paste, target_filename)) + else: + shutil.copyfile(target_file, os.path.join(self.device_location, location_to_paste)) + def install_module_mpy(self, bundle, metadata): """ :param bundle library bundle. diff --git a/circup/command_utils.py b/circup/command_utils.py index d495673..482cb4f 100644 --- a/circup/command_utils.py +++ b/circup/command_utils.py @@ -471,10 +471,13 @@ def get_dependencies(*requested_libraries, mod_names, to_install=()): _requested_libraries.append(canonical_lib_name) except KeyError: if canonical_lib_name not in WARNING_IGNORE_MODULES: - click.secho( - f"WARNING:\n\t{canonical_lib_name} is not a known CircuitPython library.", - fg="yellow", - ) + if os.path.exists(canonical_lib_name): + _requested_libraries.append(canonical_lib_name) + else: + click.secho( + f"WARNING:\n\t{canonical_lib_name} is not a known CircuitPython library.", + fg="yellow", + ) if not _requested_libraries: # If nothing is requested, we're done @@ -484,16 +487,20 @@ def get_dependencies(*requested_libraries, mod_names, to_install=()): if library not in _to_install: _to_install = _to_install + (library,) # get the requirements.txt from bundle - bundle = mod_names[library]["bundle"] - requirements_txt = bundle.requirements_for(library) - if requirements_txt: - _requested_libraries.extend( - libraries_from_requirements(requirements_txt) - ) + try: + bundle = mod_names[library]["bundle"] + requirements_txt = bundle.requirements_for(library) + if requirements_txt: + _requested_libraries.extend( + libraries_from_requirements(requirements_txt) + ) - circup_dependencies = get_circup_dependencies(bundle, library) - for circup_dependency in circup_dependencies: - _requested_libraries.append(circup_dependency) + circup_dependencies = get_circup_dependencies(bundle, library) + for circup_dependency in circup_dependencies: + _requested_libraries.append(circup_dependency) + except KeyError: + # don't check local file for further dependencies + pass # we've processed this library, remove it from the list _requested_libraries.remove(library) From e1166b6169fd0491649b36ceca511573f23e756c Mon Sep 17 00:00:00 2001 From: foamyguy Date: Sat, 11 May 2024 12:06:28 -0500 Subject: [PATCH 2/4] web workflow local install --- circup/backends.py | 45 +++++++++++++++++++++++++++-------------- circup/command_utils.py | 4 +++- 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/circup/backends.py b/circup/backends.py index 310e877..004cb65 100644 --- a/circup/backends.py +++ b/circup/backends.py @@ -89,14 +89,13 @@ def install_module_mpy(self, bundle, metadata): To be overridden by subclass """ raise NotImplementedError - + def copy_file(self, target_file, location_to_paste): - """Paste a copy of the specified file at the location given""" - """ + """Paste a copy of the specified file at the location given To be overridden by subclass """ raise NotImplementedError - + # pylint: disable=too-many-locals,too-many-branches,too-many-arguments,too-many-nested-blocks def install_module( self, device_path, device_modules, name, pyext, mod_names, upgrade=False @@ -142,20 +141,19 @@ def install_module( _metadata = _mod_names[name] module_path = _metadata["path"] self.uninstall(device_path, module_path) - + new_module_size = 0 + library_path = ( + os.path.join(device_path, self.LIB_DIR_PATH) + if not isinstance(self, WebBackend) + else urljoin(device_path, self.LIB_DIR_PATH) + ) if local_path is None: - library_path = ( - os.path.join(device_path, self.LIB_DIR_PATH) - if not isinstance(self, WebBackend) - else urljoin(device_path, self.LIB_DIR_PATH) - ) metadata = mod_names[name] bundle = metadata["bundle"] else: - library_path = local_path metadata = {"path": local_path} - + new_module_size = os.path.getsize(metadata["path"]) if os.path.isdir(metadata["path"]): # pylint: disable=unused-variable @@ -196,7 +194,7 @@ def install_module( # Use pre-compiled mpy modules. self.install_module_mpy(bundle, metadata) else: - self.copy_file(library_path, "lib") + self.copy_file(metadata["path"], "lib") click.echo("Installed '{}'.".format(name)) else: click.echo("Unknown module named, '{}'.".format(name)) @@ -542,6 +540,17 @@ def _create_library_directory(self, device_path, library_path): _writeable_error() r.raise_for_status() + def copy_file(self, target_file, location_to_paste): + if os.path.isdir(target_file): + create_directory_url = urljoin( + self.device_location, + "/".join(("fs", location_to_paste, target_file, "")), + ) + self._create_library_directory(self.device_location, create_directory_url) + self.install_dir_http(target_file) + else: + self.install_file_http(target_file) + def install_module_mpy(self, bundle, metadata): """ :param bundle library bundle. @@ -800,9 +809,15 @@ def _create_library_directory(self, device_path, library_path): def copy_file(self, target_file, location_to_paste): target_filename = target_file.split(os.path.sep)[-1] if os.path.isdir(target_file): - shutil.copytree(target_file, os.path.join(self.device_location, location_to_paste, target_filename)) + shutil.copytree( + target_file, + os.path.join(self.device_location, location_to_paste, target_filename), + ) else: - shutil.copyfile(target_file, os.path.join(self.device_location, location_to_paste)) + shutil.copyfile( + target_file, + os.path.join(self.device_location, location_to_paste, target_filename), + ) def install_module_mpy(self, bundle, metadata): """ diff --git a/circup/command_utils.py b/circup/command_utils.py index 482cb4f..71344f1 100644 --- a/circup/command_utils.py +++ b/circup/command_utils.py @@ -446,6 +446,7 @@ def get_dependencies(*requested_libraries, mod_names, to_install=()): :param list(str) to_install: Modules already selected for installation. :return: tuple of module names to install which we build """ + # pylint: disable=too-many-branches # Internal variables _to_install = to_install _requested_libraries = [] @@ -475,7 +476,8 @@ def get_dependencies(*requested_libraries, mod_names, to_install=()): _requested_libraries.append(canonical_lib_name) else: click.secho( - f"WARNING:\n\t{canonical_lib_name} is not a known CircuitPython library.", + f"WARNING:\n\t{canonical_lib_name} " + f"is not a known CircuitPython library.", fg="yellow", ) From 3d66b08d1528483d32106779be95a5e0af6219c0 Mon Sep 17 00:00:00 2001 From: foamyguy Date: Sat, 11 May 2024 12:08:38 -0500 Subject: [PATCH 3/4] update install help comment --- circup/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circup/commands.py b/circup/commands.py index ce8280e..179956e 100644 --- a/circup/commands.py +++ b/circup/commands.py @@ -302,7 +302,7 @@ def install( """ Install a named module(s) onto the device. Multiple modules can be installed at once by providing more than one module name, each - separated by a space. + separated by a space. Modules can be from a Bundle or local filepaths. """ # TODO: Ensure there's enough space on the device From 982911c04910ce0d303ba0a8f76f72058cc16faf Mon Sep 17 00:00:00 2001 From: foamyguy Date: Sun, 12 May 2024 09:50:40 -0500 Subject: [PATCH 4/4] install tab completion for local files. return early if no name. avoid multiple exists checks. --- circup/backends.py | 17 ++++++++++------- circup/command_utils.py | 2 ++ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/circup/backends.py b/circup/backends.py index 004cb65..914e0b5 100644 --- a/circup/backends.py +++ b/circup/backends.py @@ -96,7 +96,7 @@ def copy_file(self, target_file, location_to_paste): """ raise NotImplementedError - # pylint: disable=too-many-locals,too-many-branches,too-many-arguments,too-many-nested-blocks + # pylint: disable=too-many-locals,too-many-branches,too-many-arguments,too-many-nested-blocks,too-many-statements def install_module( self, device_path, device_modules, name, pyext, mod_names, upgrade=False ): # pragma: no cover @@ -116,14 +116,17 @@ def install_module( :param bool upgrade: Upgrade the specified modules if they're already installed. """ local_path = None + if os.path.exists(name): + # local file exists use that. + local_path = name + name = local_path.split(os.path.sep)[-1] + name = name.replace(".py", "").replace(".mpy", "") + click.echo(f"Installing from local path: {local_path}") + if not name: click.echo("No module name(s) provided.") - elif name in mod_names or os.path.exists(name): - if os.path.exists(name): - # local file exists use that. - local_path = name - name = local_path.split(os.path.sep)[-1] - name = name.replace(".py", "").replace(".mpy", "") + return + if name in mod_names or local_path is not None: # Grab device modules to check if module already installed if name in device_modules: diff --git a/circup/command_utils.py b/circup/command_utils.py index 71344f1..e366736 100644 --- a/circup/command_utils.py +++ b/circup/command_utils.py @@ -6,6 +6,7 @@ """ import ctypes +import glob import os from subprocess import check_output @@ -90,6 +91,7 @@ def completion_for_install(ctx, param, incomplete): module_names = {m.replace(".py", "") for m in available_modules} if incomplete: module_names = [name for name in module_names if name.startswith(incomplete)] + module_names.extend(glob.glob(f"{incomplete}*")) return sorted(module_names)