From 71a090fdd09d48b5eddeb20934ad958f89fdb814 Mon Sep 17 00:00:00 2001 From: Josef-MrBeam Date: Thu, 17 Mar 2022 11:27:20 +0100 Subject: [PATCH 01/11] copy pasta update script of mrbeam plugin --- .../scripts/update_script.py | 523 ++++++++++++++++++ 1 file changed, 523 insertions(+) create mode 100644 octoprint_netconnectd/scripts/update_script.py diff --git a/octoprint_netconnectd/scripts/update_script.py b/octoprint_netconnectd/scripts/update_script.py new file mode 100644 index 0000000..6a13545 --- /dev/null +++ b/octoprint_netconnectd/scripts/update_script.py @@ -0,0 +1,523 @@ +from __future__ import absolute_import, division, print_function + +import logging +import os +import re +import shutil +import subprocess +import sys +from io import BytesIO + +import yaml +import zipfile +import requests +from octoprint.plugins.softwareupdate import exceptions +from octoprint.plugins.softwareupdate.updaters.pip import _get_pip_caller + +from octoprint.settings import _default_basedir + +from octoprint_mrbeam.util.pip_util import get_version_of_pip_module #TODO check how to be independent of mrbeam plugin +from requests.adapters import HTTPAdapter +from urllib3 import Retry +from urllib3.exceptions import MaxRetryError, ConnectionError + +_logger = logging.getLogger("octoprint.plugins.netconnectd.softwareupdate.updatescript") + +UPDATE_CONFIG_NAME = "netconnectd_plugin" +REPO_NAME = "OctoPrint-Netconnectd" +MAIN_SRC_FOLDER_NAME = "octoprint_netconnectd" +PLUGIN_NAME = "OctoPrint-Netconnectd" +DEFAULT_OPRINT_VENV = "/home/pi/oprint/bin/pip" + +""" +copy pasta of mrbeam plugin update script +""" + +def _parse_arguments(): + import argparse + + boolean_trues = ["true", "yes", "1"] + + parser = argparse.ArgumentParser(prog="update_script.py") + + parser.add_argument( + "--git", + action="store", + type=str, + dest="git_executable", + help="Specify git executable to use", + ) + parser.add_argument( + "--python", + action="store", + type=str, + dest="python_executable", + help="Specify python executable to use", + ) + parser.add_argument( + "--force", + action="store", + type=lambda x: x in boolean_trues, + dest="force", + default=False, + help="Set this to true to force the update to only the specified version (nothing newer, nothing older)", + ) + parser.add_argument( + "--sudo", action="store_true", dest="sudo", help="Install with sudo" + ) + parser.add_argument( + "--user", + action="store_true", + dest="user", + help="Install to the user site directory instead of the general site directory", + ) + parser.add_argument( + "--branch", + action="store", + type=str, + dest="branch", + default=None, + help="Specify the branch to make sure is checked out", + ) + parser.add_argument( + "--call", + action="store", + type=lambda x: x in boolean_trues, + dest="call", + default=False, + ) + parser.add_argument( + "--archive", + action="store", + type=str, + dest="archive", + default=None, + help="path of target zip file on local system", + ) + parser.add_argument( + "--target_version", + action="store", + type=str, + dest="target_version", + default=None, + help="version number of the target", + ) + parser.add_argument( + "folder", + type=str, + help="Specify the base folder of the OctoPrint installation to update", + ) + parser.add_argument( + "target", type=str, help="Specify the commit or tag to which to update" + ) + + args = parser.parse_args() + + return args + + +def get_dependencies(path): + """ + return the dependencies saved in the + + Args: + path: path to the dependencies.txt file + + Returns: + list of dependencie dict [{"name", "version"}] + """ + dependencies_path = os.path.join(path, "dependencies.txt") + dependencies_pattern = r"([a-z]+(?:[_-][a-z]+)*)(.=)+([0-9]+.[0-9]+.[0-9]+)" + try: + with open(dependencies_path, "r") as f: + dependencies_content = f.read() + dependencies = re.findall(dependencies_pattern, dependencies_content) + dependencies = [{"name": dep[0], "version": dep[2]} for dep in dependencies] + except IOError: + raise RuntimeError("Could not load dependencies") + return dependencies + + +def get_update_info(): + """ + returns the update info saved in the update_info.json file + """ + update_info_path = os.path.join(_default_basedir("OctoPrint"), "update_info.json") + try: + with open(update_info_path, "r") as f: + update_info = yaml.safe_load(f) + except IOError: + raise RuntimeError("Could not load update info") + except ValueError: + raise RuntimeError("update info not valid json") + except yaml.YAMLError as e: + raise RuntimeError("update info not valid json - {}".format(e)) + return update_info + + +def build_wheels(queue): + """ + build the wheels of the packages in the queue + + Args: + queue: dict of venvs with a list of packages to build the wheels + + Returns: + None + + """ + for venv, packages in queue.items(): + + pip_caller = _get_pip_caller(command=venv) + if pip_caller is None: + raise exceptions.UpdateError("Can't run pip", None) + + def _log_call(*lines): + _log(lines, prefix=" ", stream="call") + + def _log_stdout(*lines): + _log(lines, prefix=">", stream="stdout") + + def _log_stderr(*lines): + _log(lines, prefix="!", stream="stderr") + + def _log(lines, prefix=None, stream=None, strip=True): + if strip: + lines = map(lambda x: x.strip(), lines) + for line in lines: + print(u"{} {}".format(prefix, line)) + + if _logger is not None: + pip_caller.on_log_call = _log_call + pip_caller.on_log_stdout = _log_stdout + pip_caller.on_log_stderr = _log_stderr + + pip_args = [ + "wheel", + "--no-python-version-warning", + "--disable-pip-version-check", + "--wheel-dir=/tmp/wheelhouse", # Build wheels into , where the default is the current working directory. + "--no-dependencies", # Don't install package dependencies. + ] + for package in packages: + if package.get("archive"): + pip_args.append(package.get("archive")) + else: + raise exceptions.UpdateError( + "Archive not found for package {}".format(package) + ) + + returncode, stdout, stderr = pip_caller.execute(*pip_args) + if returncode != 0: + raise exceptions.UpdateError( + "Error while executing pip wheel", (stdout, stderr) + ) + + +def install_wheels(queue): + """ + installs the wheels in the given venv of the queue + + Args: + queue: dict of venvs with a list of packages to install + + Returns: + None + """ + for venv, packages in queue.items(): + + pip_caller = _get_pip_caller(command=venv) + if pip_caller is None: + raise exceptions.UpdateError("Can't run pip", None) + + def _log_call(*lines): + _log(lines, prefix=" ", stream="call") + + def _log_stdout(*lines): + _log(lines, prefix=">", stream="stdout") + + def _log_stderr(*lines): + _log(lines, prefix="!", stream="stderr") + + def _log(lines, prefix=None, stream=None, strip=True): + if strip: + lines = map(lambda x: x.strip(), lines) + for line in lines: + print(u"{} {}".format(prefix, line)) + + if _logger is not None: + pip_caller.on_log_call = _log_call + pip_caller.on_log_stdout = _log_stdout + pip_caller.on_log_stderr = _log_stderr + + pip_args = [ + "install", + "--no-python-version-warning", + "--disable-pip-version-check", + "--upgrade", # Upgrade all specified packages to the newest available version. The handling of dependencies depends on the upgrade-strategy used. + "--no-index", # Ignore package index (only looking at --find-links URLs instead). + "--find-links=/tmp/wheelhouse", # If a URL or path to an html file, then parse for links to archives such as sdist (.tar.gz) or wheel (.whl) files. If a local path or file:// URL that's a directory, then look for archives in the directory listing. Links to VCS project URLs are not supported. + "--no-dependencies", # Don't install package dependencies. + ] + for package in packages: + pip_args.append( + "{package}=={package_version}".format( + package=package["name"], package_version=package["target"] + ) + ) + + returncode, stdout, stderr = pip_caller.execute(*pip_args) + if returncode != 0: + raise exceptions.UpdateError( + "Error while executing pip install", (stdout, stderr) + ) + + +def build_queue(update_info, dependencies, target, plugin_archive): + """ + build the queue of packages to install + + Args: + update_info: a dict of informations how to update the packages + dependencies: a list dicts of dependencies [{"name", "version"}] + target: target of the Mr Beam Plugin to update to + + Returns: + install_queue: dict of venvs with a list of package dicts {"": [{"name", "archive", "target"}] + """ + install_queue = {} + + install_queue.setdefault( + update_info.get(UPDATE_CONFIG_NAME).get("pip_command", DEFAULT_OPRINT_VENV), [] + ).append( + { + "name": PLUGIN_NAME, + "archive": plugin_archive, + "target": target, + } + ) + + if dependencies: + for dependency in dependencies: + plugin_config = update_info.get(UPDATE_CONFIG_NAME) + plugin_dependencies_config = plugin_config.get("dependencies") + dependency_config = plugin_dependencies_config.get(dependency["name"]) + + # fail if requirements file contains dependencies but cloud config not + if dependency_config == None: + raise exceptions.UpdateError( + "no update info for dependency", dependency["name"] + ) + if dependency_config.get("pip"): + archive = dependency_config["pip"].format( + repo=dependency_config["repo"], + user=dependency_config["user"], + target_version="v{version}".format(version=dependency["version"]), + ) + else: + raise exceptions.UpdateError( + "pip not configured for {}".format(dependency["name"]) + ) + + version = get_version_of_pip_module( + dependency["name"], + dependency_config.get("pip_command", DEFAULT_OPRINT_VENV), + ) + if version != dependency["version"]: + install_queue.setdefault( + dependency_config.get("pip_command", DEFAULT_OPRINT_VENV), [] + ).append( + { + "name": dependency["name"], + "archive": archive, + "target": dependency["version"], + } + ) + return install_queue + + +def run_update(): + """ + collects the dependencies and the update info, builds the wheels and installs them in the correct venv + """ + + args = _parse_arguments() + + # get dependencies + dependencies = get_dependencies(args.folder) + + # get update config of dependencies + update_info = get_update_info() + + install_queue = build_queue( + update_info, dependencies, args.target_version, args.archive + ) + + print("install_queue", install_queue) + if install_queue is not None: + build_wheels(install_queue) + install_wheels(install_queue) + + +def retryget(url, retrys=3, backoff_factor=0.3): + """ + retrys the get times + + Args: + url: url to access + retrys: number of retrys + backoff_factor: factor for time between retrys + + Returns: + response + """ + try: + s = requests.Session() + retry = Retry(connect=retrys, backoff_factor=backoff_factor) + adapter = HTTPAdapter(max_retries=retry) + s.mount("https://", adapter) + s.keep_alive = False + + response = s.request("GET", url) + return response + except MaxRetryError: + raise RuntimeError("timeout while trying to get {}".format(url)) + except ConnectionError: + raise RuntimeError("connection error while trying to get {}".format(url)) + + +def loadPluginTarget(archive, folder): + """ + download the archive of the Plugin and copy dependencies and update script in the working directory + + Args: + archive: path of the archive to download and unzip + folder: working directory + + Returns: + (zip_file_path, target_version) - path of the downloaded zip file and target version string + """ + + # download target repo zip + req = retryget(archive) + filename = archive.split("/")[-1] + zip_file_path = os.path.join(folder, filename) + try: + with open(zip_file_path, "wb") as output_file: + output_file.write(req.content) + except IOError: + raise RuntimeError( + "Could not save the zip file to the working directory {}".format(folder) + ) + + # unzip repo + plugin_extracted_path = os.path.join(folder, UPDATE_CONFIG_NAME) + plugin_extracted_path_folder = os.path.join( + plugin_extracted_path, + "{repo_name}-{target}".format(repo_name=REPO_NAME, target=filename.split(".zip")[0]), + ) + try: + plugin_zipfile = zipfile.ZipFile(BytesIO(req.content)) + plugin_zipfile.extractall(plugin_extracted_path) + plugin_zipfile.close() + except (zipfile.BadZipfile, zipfile.LargeZipFile) as e: + raise RuntimeError("Could not unzip plugin repo - error: {}".format(e)) + + # copy new dependencies to working directory + try: + shutil.copy2( + os.path.join( + plugin_extracted_path_folder, MAIN_SRC_FOLDER_NAME, "dependencies.txt" + ), + os.path.join(folder, "dependencies.txt"), + ) + except IOError: + raise RuntimeError("Could not copy dependencies to working directory") + + # copy new update script to working directory + try: + shutil.copy2( + os.path.join( + plugin_extracted_path_folder, + MAIN_SRC_FOLDER_NAME, + "scripts/update_script.py", + ), + os.path.join(folder, "update_script.py"), + ) + except IOError: + raise RuntimeError("Could not copy update_script to working directory") + shutil.copy2( + os.path.join(os.path.dirname(os.path.abspath(__file__)), "update_script.py"), + os.path.join(folder, "update_script.py"), + ) # todo only for debug + + # get target version + exec( + open( + os.path.join( + plugin_extracted_path_folder, MAIN_SRC_FOLDER_NAME, "__version.py" + ) + ).read() + ) + target_version = __version__ + + return zip_file_path, target_version + + +def main(): + """ + loads the dependencies.txt and the update_script of the given target and executes the new update_script + + Args: + target: target of the Mr Beam Plugin to update to + call: if true executet the update itselfe + """ + + args = _parse_arguments() + if args.call: + if args.archive is None or args.target_version is None: + raise RuntimeError( + "Could not run update archive or target_version is missing" + ) + run_update() + else: + + folder = args.folder + + import os + + if not os.access(folder, os.W_OK): + raise RuntimeError("Could not update, base folder is not writable") + + update_info = get_update_info() + archive, target_version = loadPluginTarget( + update_info.get(UPDATE_CONFIG_NAME).get("pip").format(target_version=args.target), + folder, + ) + + # call new update script with args + sys.argv = [ + "--call=true", + "--archive={}".format(archive), + "--target_version={}".format(target_version), + ] + sys.argv[1:] + try: + result = subprocess.call( + [sys.executable, os.path.join(folder, "update_script.py")] + + sys.argv, + stderr=subprocess.STDOUT, + ) + except subprocess.CalledProcessError as e: + print(e.output) + raise RuntimeError("error code %s", (e.returncode, e.output)) + + if result != 0: + raise exceptions.UpdateError("Error Could not update", result) + + raise exceptions.UpdateError( + "Error Could not update", result + ) # TODO only for debug + + +if __name__ == "__main__": + main() From 5e78db650cc67d459b67ffb079b5ccbc147726fb Mon Sep 17 00:00:00 2001 From: Josef-MrBeam Date: Thu, 17 Mar 2022 11:27:38 +0100 Subject: [PATCH 02/11] prepare __version file --- octoprint_netconnectd/__version.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 octoprint_netconnectd/__version.py diff --git a/octoprint_netconnectd/__version.py b/octoprint_netconnectd/__version.py new file mode 100644 index 0000000..3fd5f1e --- /dev/null +++ b/octoprint_netconnectd/__version.py @@ -0,0 +1 @@ +__version__ = "TODO" \ No newline at end of file From 5f9d6753d8ccb25f0e335708152a1962a095bcc7 Mon Sep 17 00:00:00 2001 From: Josef-MrBeam Date: Thu, 17 Mar 2022 11:27:48 +0100 Subject: [PATCH 03/11] add dependencies --- octoprint_netconnectd/dependencies.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 octoprint_netconnectd/dependencies.txt diff --git a/octoprint_netconnectd/dependencies.txt b/octoprint_netconnectd/dependencies.txt new file mode 100644 index 0000000..8ba4c95 --- /dev/null +++ b/octoprint_netconnectd/dependencies.txt @@ -0,0 +1 @@ +netconnectd==0.1.0 \ No newline at end of file From 42875deefec5dfdc73ab7b56ef34c5664fcc96d1 Mon Sep 17 00:00:00 2001 From: Josef-MrBeam Date: Thu, 17 Mar 2022 16:33:10 +0100 Subject: [PATCH 04/11] change dependencie regex to symtantic versioning set version in version file --- octoprint_netconnectd/__version.py | 2 +- octoprint_netconnectd/scripts/update_script.py | 17 ++++++++--------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/octoprint_netconnectd/__version.py b/octoprint_netconnectd/__version.py index 3fd5f1e..ade5f41 100644 --- a/octoprint_netconnectd/__version.py +++ b/octoprint_netconnectd/__version.py @@ -1 +1 @@ -__version__ = "TODO" \ No newline at end of file +__version__ = "0.1.10" #TODO fix with versioneer \ No newline at end of file diff --git a/octoprint_netconnectd/scripts/update_script.py b/octoprint_netconnectd/scripts/update_script.py index 6a13545..beba0b8 100644 --- a/octoprint_netconnectd/scripts/update_script.py +++ b/octoprint_netconnectd/scripts/update_script.py @@ -127,7 +127,7 @@ def get_dependencies(path): list of dependencie dict [{"name", "version"}] """ dependencies_path = os.path.join(path, "dependencies.txt") - dependencies_pattern = r"([a-z]+(?:[_-][a-z]+)*)(.=)+([0-9]+.[0-9]+.[0-9]+)" + dependencies_pattern = r"([a-z]+(?:[_-][a-z]+)*)(.=)+((0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?)" try: with open(dependencies_path, "r") as f: dependencies_content = f.read() @@ -414,7 +414,9 @@ def loadPluginTarget(archive, folder): plugin_extracted_path = os.path.join(folder, UPDATE_CONFIG_NAME) plugin_extracted_path_folder = os.path.join( plugin_extracted_path, - "{repo_name}-{target}".format(repo_name=REPO_NAME, target=filename.split(".zip")[0]), + "{repo_name}-{target}".format( + repo_name=REPO_NAME, target=filename.split(".zip")[0] + ), ) try: plugin_zipfile = zipfile.ZipFile(BytesIO(req.content)) @@ -491,7 +493,9 @@ def main(): update_info = get_update_info() archive, target_version = loadPluginTarget( - update_info.get(UPDATE_CONFIG_NAME).get("pip").format(target_version=args.target), + update_info.get(UPDATE_CONFIG_NAME) + .get("pip") + .format(target_version=args.target), folder, ) @@ -503,8 +507,7 @@ def main(): ] + sys.argv[1:] try: result = subprocess.call( - [sys.executable, os.path.join(folder, "update_script.py")] - + sys.argv, + [sys.executable, os.path.join(folder, "update_script.py")] + sys.argv, stderr=subprocess.STDOUT, ) except subprocess.CalledProcessError as e: @@ -514,10 +517,6 @@ def main(): if result != 0: raise exceptions.UpdateError("Error Could not update", result) - raise exceptions.UpdateError( - "Error Could not update", result - ) # TODO only for debug - if __name__ == "__main__": main() From f9a26f4029a76c5a374dee0cb8c7cd653976ccc4 Mon Sep 17 00:00:00 2001 From: Josef-MrBeam Date: Thu, 17 Mar 2022 16:39:11 +0100 Subject: [PATCH 05/11] remove debug --- octoprint_netconnectd/scripts/update_script.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/octoprint_netconnectd/scripts/update_script.py b/octoprint_netconnectd/scripts/update_script.py index beba0b8..57272b5 100644 --- a/octoprint_netconnectd/scripts/update_script.py +++ b/octoprint_netconnectd/scripts/update_script.py @@ -448,10 +448,6 @@ def loadPluginTarget(archive, folder): ) except IOError: raise RuntimeError("Could not copy update_script to working directory") - shutil.copy2( - os.path.join(os.path.dirname(os.path.abspath(__file__)), "update_script.py"), - os.path.join(folder, "update_script.py"), - ) # todo only for debug # get target version exec( From f6216a5db84b4b328c5d89734352496b40664f4e Mon Sep 17 00:00:00 2001 From: Josef-MrBeam Date: Mon, 21 Mar 2022 16:33:38 +0100 Subject: [PATCH 06/11] copy over changes from MrBeam Plugin update script --- .../scripts/update_script.py | 132 +++++++----------- 1 file changed, 52 insertions(+), 80 deletions(-) diff --git a/octoprint_netconnectd/scripts/update_script.py b/octoprint_netconnectd/scripts/update_script.py index 57272b5..bb10f66 100644 --- a/octoprint_netconnectd/scripts/update_script.py +++ b/octoprint_netconnectd/scripts/update_script.py @@ -1,22 +1,24 @@ from __future__ import absolute_import, division, print_function +import argparse +import json import logging import os import re import shutil import subprocess import sys -from io import BytesIO - -import yaml import zipfile import requests -from octoprint.plugins.softwareupdate import exceptions -from octoprint.plugins.softwareupdate.updaters.pip import _get_pip_caller +from io import BytesIO + +from octoprint.plugins.softwareupdate import exceptions from octoprint.settings import _default_basedir -from octoprint_mrbeam.util.pip_util import get_version_of_pip_module #TODO check how to be independent of mrbeam plugin +from octoprint_mrbeam.util.pip_util import get_version_of_pip_module, \ + get_pip_caller # TODO check how to be independent of mrbeam plugin + from requests.adapters import HTTPAdapter from urllib3 import Retry from urllib3.exceptions import MaxRetryError, ConnectionError @@ -33,12 +35,11 @@ copy pasta of mrbeam plugin update script """ -def _parse_arguments(): - import argparse +def _parse_arguments(): boolean_trues = ["true", "yes", "1"] - parser = argparse.ArgumentParser(prog="update_script.py") + parser = argparse.ArgumentParser(prog=__file__) parser.add_argument( "--git", @@ -85,6 +86,7 @@ def _parse_arguments(): type=lambda x: x in boolean_trues, dest="call", default=False, + help="Calls the update methode", ) parser.add_argument( "--archive", @@ -92,7 +94,7 @@ def _parse_arguments(): type=str, dest="archive", default=None, - help="path of target zip file on local system", + help="Path of target zip file on local system", ) parser.add_argument( "--target_version", @@ -100,7 +102,7 @@ def _parse_arguments(): type=str, dest="target_version", default=None, - help="version number of the target", + help="Version number of the target", ) parser.add_argument( "folder", @@ -128,6 +130,15 @@ def get_dependencies(path): """ dependencies_path = os.path.join(path, "dependencies.txt") dependencies_pattern = r"([a-z]+(?:[_-][a-z]+)*)(.=)+((0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?)" + """ + Example: + input: iobeam==0.7.15 + mrb-hw-info==0.0.25 + mrbeam-ledstrips==0.2.2-alpha.2 + output: [[iobeam][==][0.7.15]] + [[mrb-hw-info][==][0.0.25]] + [[mrbeam-ledstrips][==][0.2.2-alpha.2]] + """ try: with open(dependencies_path, "r") as f: dependencies_content = f.read() @@ -145,52 +156,26 @@ def get_update_info(): update_info_path = os.path.join(_default_basedir("OctoPrint"), "update_info.json") try: with open(update_info_path, "r") as f: - update_info = yaml.safe_load(f) + update_info = json.load(f) except IOError: raise RuntimeError("Could not load update info") - except ValueError: - raise RuntimeError("update info not valid json") - except yaml.YAMLError as e: + except ValueError as e: raise RuntimeError("update info not valid json - {}".format(e)) return update_info -def build_wheels(queue): +def build_wheels(build_queue): """ build the wheels of the packages in the queue Args: - queue: dict of venvs with a list of packages to build the wheels + build_queue: dict of venvs with a list of packages to build the wheels Returns: None """ - for venv, packages in queue.items(): - - pip_caller = _get_pip_caller(command=venv) - if pip_caller is None: - raise exceptions.UpdateError("Can't run pip", None) - - def _log_call(*lines): - _log(lines, prefix=" ", stream="call") - - def _log_stdout(*lines): - _log(lines, prefix=">", stream="stdout") - - def _log_stderr(*lines): - _log(lines, prefix="!", stream="stderr") - - def _log(lines, prefix=None, stream=None, strip=True): - if strip: - lines = map(lambda x: x.strip(), lines) - for line in lines: - print(u"{} {}".format(prefix, line)) - - if _logger is not None: - pip_caller.on_log_call = _log_call - pip_caller.on_log_stdout = _log_stdout - pip_caller.on_log_stderr = _log_stderr + for venv, packages in build_queue.items(): pip_args = [ "wheel", @@ -203,52 +188,31 @@ def _log(lines, prefix=None, stream=None, strip=True): if package.get("archive"): pip_args.append(package.get("archive")) else: - raise exceptions.UpdateError( - "Archive not found for package {}".format(package) - ) + raise RuntimeError("Archive not found for package {}".format(package)) - returncode, stdout, stderr = pip_caller.execute(*pip_args) + returncode, exec_stdout, exec_stderr = get_pip_caller(venv, _logger).execute( + *pip_args + ) if returncode != 0: raise exceptions.UpdateError( - "Error while executing pip wheel", (stdout, stderr) + "Error while executing pip wheel", (exec_stdout, exec_stderr) ) -def install_wheels(queue): +def install_wheels(install_queue): """ installs the wheels in the given venv of the queue Args: - queue: dict of venvs with a list of packages to install + install_queue: dict of venvs with a list of packages to install Returns: None """ - for venv, packages in queue.items(): - - pip_caller = _get_pip_caller(command=venv) - if pip_caller is None: - raise exceptions.UpdateError("Can't run pip", None) - - def _log_call(*lines): - _log(lines, prefix=" ", stream="call") + if not isinstance(install_queue, dict): + raise RuntimeError("install queue is not a dict") - def _log_stdout(*lines): - _log(lines, prefix=">", stream="stdout") - - def _log_stderr(*lines): - _log(lines, prefix="!", stream="stderr") - - def _log(lines, prefix=None, stream=None, strip=True): - if strip: - lines = map(lambda x: x.strip(), lines) - for line in lines: - print(u"{} {}".format(prefix, line)) - - if _logger is not None: - pip_caller.on_log_call = _log_call - pip_caller.on_log_stdout = _log_stdout - pip_caller.on_log_stderr = _log_stderr + for venv, packages in install_queue.items(): pip_args = [ "install", @@ -266,10 +230,12 @@ def _log(lines, prefix=None, stream=None, strip=True): ) ) - returncode, stdout, stderr = pip_caller.execute(*pip_args) + returncode, exec_stdout, exec_stderr = get_pip_caller(venv, _logger).execute( + *pip_args + ) if returncode != 0: raise exceptions.UpdateError( - "Error while executing pip install", (stdout, stderr) + "Error while executing pip install", (exec_stdout, exec_stderr) ) @@ -296,7 +262,7 @@ def build_queue(update_info, dependencies, target, plugin_archive): "target": target, } ) - + print("dependencies - {}".format(dependencies)) if dependencies: for dependency in dependencies: plugin_config = update_info.get(UPDATE_CONFIG_NAME) @@ -305,8 +271,8 @@ def build_queue(update_info, dependencies, target, plugin_archive): # fail if requirements file contains dependencies but cloud config not if dependency_config == None: - raise exceptions.UpdateError( - "no update info for dependency", dependency["name"] + raise RuntimeError( + "no update info for dependency {}".format(dependency["name"]) ) if dependency_config.get("pip"): archive = dependency_config["pip"].format( @@ -315,7 +281,7 @@ def build_queue(update_info, dependencies, target, plugin_archive): target_version="v{version}".format(version=dependency["version"]), ) else: - raise exceptions.UpdateError( + raise RuntimeError( "pip not configured for {}".format(dependency["name"]) ) @@ -333,6 +299,12 @@ def build_queue(update_info, dependencies, target, plugin_archive): "target": dependency["version"], } ) + else: + print( + "skip dependency {} as the target version {} is already installed".format( + dependency["name"], dependency["version"] + ) + ) return install_queue @@ -511,7 +483,7 @@ def main(): raise RuntimeError("error code %s", (e.returncode, e.output)) if result != 0: - raise exceptions.UpdateError("Error Could not update", result) + raise RuntimeError("Error Could not update returncode - {}".format(result)) if __name__ == "__main__": From 8f6d47e7c2a780f93eae76d8949b7c41a5657517 Mon Sep 17 00:00:00 2001 From: Josef-MrBeam Date: Mon, 21 Mar 2022 16:45:30 +0100 Subject: [PATCH 07/11] added test for dependency file --- tests/__init__.py | 0 tests/test_dependencies.py | 16 ++++++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 tests/__init__.py create mode 100644 tests/test_dependencies.py diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py new file mode 100644 index 0000000..8d39811 --- /dev/null +++ b/tests/test_dependencies.py @@ -0,0 +1,16 @@ +import os +import re +import unittest + + +class TestDependencies(unittest.TestCase): + def test_dependencies_file(self): + dependencies_path = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "../octoprint_netconnectd/dependencies.txt", + ) + dependencies_pattern = r"([a-z]+(?:[_-][a-z]+)*)==((0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?)" + with open(dependencies_path, "r") as f: + lines = f.readlines() + for line in lines: + self.assertRegexpMatches(line, dependencies_pattern) From 61b90f9196c024d42181e907c41e7dbbeed4633e Mon Sep 17 00:00:00 2001 From: Josef-MrBeam Date: Thu, 24 Mar 2022 17:09:07 +0100 Subject: [PATCH 08/11] copied the changes from the Mr Beam Plugin update script PR https://github.com/mrbeam/MrBeamPlugin/pull/1438 --- .../scripts/update_script.py | 62 +++++++++---------- 1 file changed, 28 insertions(+), 34 deletions(-) diff --git a/octoprint_netconnectd/scripts/update_script.py b/octoprint_netconnectd/scripts/update_script.py index bb10f66..416c66c 100644 --- a/octoprint_netconnectd/scripts/update_script.py +++ b/octoprint_netconnectd/scripts/update_script.py @@ -30,6 +30,7 @@ MAIN_SRC_FOLDER_NAME = "octoprint_netconnectd" PLUGIN_NAME = "OctoPrint-Netconnectd" DEFAULT_OPRINT_VENV = "/home/pi/oprint/bin/pip" +PIP_WHEEL_TEMP_FOLDER = "/tmp/wheelhouse" """ copy pasta of mrbeam plugin update script @@ -96,14 +97,6 @@ def _parse_arguments(): default=None, help="Path of target zip file on local system", ) - parser.add_argument( - "--target_version", - action="store", - type=str, - dest="target_version", - default=None, - help="Version number of the target", - ) parser.add_argument( "folder", type=str, @@ -175,13 +168,25 @@ def build_wheels(build_queue): None """ + try: + if not os.path.isdir(PIP_WHEEL_TEMP_FOLDER): + os.mkdir(PIP_WHEEL_TEMP_FOLDER) + except OSError as e: + raise RuntimeError("can't create wheel tmp folder {} - {}".format(PIP_WHEEL_TEMP_FOLDER, e)) + for venv, packages in build_queue.items(): + tmp_folder = os.path.join(PIP_WHEEL_TEMP_FOLDER, re.search(r"\w+((?=\/venv)|(?=\/bin))", venv).group(0)) + if os.path.isdir(tmp_folder): + try: + os.system("sudo rm -r {}".format(tmp_folder)) + except Exception as e: + raise RuntimeError("can't delete pip wheel temp folder {} - {}".format(tmp_folder, e)) pip_args = [ "wheel", "--no-python-version-warning", "--disable-pip-version-check", - "--wheel-dir=/tmp/wheelhouse", # Build wheels into , where the default is the current working directory. + "--wheel-dir={}".format(tmp_folder), # Build wheels into , where the default is the current working directory. "--no-dependencies", # Don't install package dependencies. ] for package in packages: @@ -213,20 +218,20 @@ def install_wheels(install_queue): raise RuntimeError("install queue is not a dict") for venv, packages in install_queue.items(): - + tmp_folder = os.path.join(PIP_WHEEL_TEMP_FOLDER, re.search(r"\w+((?=\/venv)|(?=\/bin))", venv).group(0)) pip_args = [ "install", "--no-python-version-warning", "--disable-pip-version-check", "--upgrade", # Upgrade all specified packages to the newest available version. The handling of dependencies depends on the upgrade-strategy used. "--no-index", # Ignore package index (only looking at --find-links URLs instead). - "--find-links=/tmp/wheelhouse", # If a URL or path to an html file, then parse for links to archives such as sdist (.tar.gz) or wheel (.whl) files. If a local path or file:// URL that's a directory, then look for archives in the directory listing. Links to VCS project URLs are not supported. + "--find-links={}".format(tmp_folder), # If a URL or path to an html file, then parse for links to archives such as sdist (.tar.gz) or wheel (.whl) files. If a local path or file:// URL that's a directory, then look for archives in the directory listing. Links to VCS project URLs are not supported. "--no-dependencies", # Don't install package dependencies. ] for package in packages: pip_args.append( - "{package}=={package_version}".format( - package=package["name"], package_version=package["target"] + "{package}".format( + package=package["name"] ) ) @@ -239,14 +244,14 @@ def install_wheels(install_queue): ) -def build_queue(update_info, dependencies, target, plugin_archive): +def build_queue(update_info, dependencies, plugin_archive): """ build the queue of packages to install Args: update_info: a dict of informations how to update the packages dependencies: a list dicts of dependencies [{"name", "version"}] - target: target of the Mr Beam Plugin to update to + plugin_archive: path to archive of the plugin Returns: install_queue: dict of venvs with a list of package dicts {"": [{"name", "archive", "target"}] @@ -259,7 +264,7 @@ def build_queue(update_info, dependencies, target, plugin_archive): { "name": PLUGIN_NAME, "archive": plugin_archive, - "target": target, + "target": '', } ) print("dependencies - {}".format(dependencies)) @@ -322,7 +327,7 @@ def run_update(): update_info = get_update_info() install_queue = build_queue( - update_info, dependencies, args.target_version, args.archive + update_info, dependencies, args.archive ) print("install_queue", install_queue) @@ -367,7 +372,7 @@ def loadPluginTarget(archive, folder): folder: working directory Returns: - (zip_file_path, target_version) - path of the downloaded zip file and target version string + zip_file_path - path of the downloaded zip file """ # download target repo zip @@ -421,17 +426,7 @@ def loadPluginTarget(archive, folder): except IOError: raise RuntimeError("Could not copy update_script to working directory") - # get target version - exec( - open( - os.path.join( - plugin_extracted_path_folder, MAIN_SRC_FOLDER_NAME, "__version.py" - ) - ).read() - ) - target_version = __version__ - - return zip_file_path, target_version + return zip_file_path def main(): @@ -445,9 +440,9 @@ def main(): args = _parse_arguments() if args.call: - if args.archive is None or args.target_version is None: + if args.archive is None: raise RuntimeError( - "Could not run update archive or target_version is missing" + "Could not run update archive is missing" ) run_update() else: @@ -460,7 +455,7 @@ def main(): raise RuntimeError("Could not update, base folder is not writable") update_info = get_update_info() - archive, target_version = loadPluginTarget( + archive = loadPluginTarget( update_info.get(UPDATE_CONFIG_NAME) .get("pip") .format(target_version=args.target), @@ -470,8 +465,7 @@ def main(): # call new update script with args sys.argv = [ "--call=true", - "--archive={}".format(archive), - "--target_version={}".format(target_version), + "--archive={}".format(archive) ] + sys.argv[1:] try: result = subprocess.call( From 342b7e34226a71d6af506eaf0329f871045b6cd8 Mon Sep 17 00:00:00 2001 From: Josef-MrBeam Date: Fri, 25 Mar 2022 09:54:26 +0100 Subject: [PATCH 09/11] fix config name --- octoprint_netconnectd/scripts/update_script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octoprint_netconnectd/scripts/update_script.py b/octoprint_netconnectd/scripts/update_script.py index 416c66c..39d5172 100644 --- a/octoprint_netconnectd/scripts/update_script.py +++ b/octoprint_netconnectd/scripts/update_script.py @@ -25,7 +25,7 @@ _logger = logging.getLogger("octoprint.plugins.netconnectd.softwareupdate.updatescript") -UPDATE_CONFIG_NAME = "netconnectd_plugin" +UPDATE_CONFIG_NAME = "netconnectd" REPO_NAME = "OctoPrint-Netconnectd" MAIN_SRC_FOLDER_NAME = "octoprint_netconnectd" PLUGIN_NAME = "OctoPrint-Netconnectd" From 29aa13ee2bac8d988eef8b68c78e7c5568efe391 Mon Sep 17 00:00:00 2001 From: Josef-MrBeam Date: Mon, 28 Mar 2022 10:34:13 +0200 Subject: [PATCH 10/11] copy needed mrbeam utils to repo --- .../scripts/update_script.py | 4 +- octoprint_netconnectd/util/__init__.py | 0 octoprint_netconnectd/util/cmd_exec.py | 78 +++++++++++++ octoprint_netconnectd/util/pip_util.py | 109 ++++++++++++++++++ 4 files changed, 189 insertions(+), 2 deletions(-) create mode 100644 octoprint_netconnectd/util/__init__.py create mode 100644 octoprint_netconnectd/util/cmd_exec.py create mode 100644 octoprint_netconnectd/util/pip_util.py diff --git a/octoprint_netconnectd/scripts/update_script.py b/octoprint_netconnectd/scripts/update_script.py index 39d5172..d98cb9e 100644 --- a/octoprint_netconnectd/scripts/update_script.py +++ b/octoprint_netconnectd/scripts/update_script.py @@ -16,8 +16,8 @@ from octoprint.plugins.softwareupdate import exceptions from octoprint.settings import _default_basedir -from octoprint_mrbeam.util.pip_util import get_version_of_pip_module, \ - get_pip_caller # TODO check how to be independent of mrbeam plugin +from octoprint_netconnectd.util.pip_util import get_version_of_pip_module, \ + get_pip_caller from requests.adapters import HTTPAdapter from urllib3 import Retry diff --git a/octoprint_netconnectd/util/__init__.py b/octoprint_netconnectd/util/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/octoprint_netconnectd/util/cmd_exec.py b/octoprint_netconnectd/util/cmd_exec.py new file mode 100644 index 0000000..57c544f --- /dev/null +++ b/octoprint_netconnectd/util/cmd_exec.py @@ -0,0 +1,78 @@ +""" +copy pasta of octoprint_mrbeam/util/cmd_exec.py +only changed the logging to be mrbeam plugin independend +""" +import logging +import subprocess +from logging import DEBUG + + +def exec_cmd(cmd, log=True, shell=True, loglvl=DEBUG): + """ + Executes a system command + :param cmd: + :return: True if system returncode was 0, + False if the command returned with an error, + None if there was an exception. + """ + _logger = logging.getLogger(__name__ + ".exec_cmd") + code = None + if log: + _logger.log(loglvl, "cmd=%s", cmd) + try: + code = subprocess.call(cmd, shell=shell) + except Exception as e: + _logger.debug( + "Failed to execute command '%s', return code: %s, Exception: %s", + cmd, + code, + e, + ) + return None + if code != 0 and log: + _logger.info("cmd= '%s', return code: '%s'", code) + return code == 0 + + +def exec_cmd_output(cmd, log=True, shell=False, loglvl=DEBUG): + """ + Executes a system command and returns its output. + :param cmd: + :return: Tuple(String:output , int return_code) + """ + _logger = logging.getLogger(__name__ + "exec_cmd_output") + output = None + code = 0 + if log: + _logger.log(loglvl, "cmd='%s'", cmd) + try: + output = subprocess.check_output(cmd, shell=shell, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + code = e.returncode + + if not log: + cmd = cmd[:50] + "..." if len(cmd) > 30 else cmd + if e.output is not None: + output = e.output[:30] + "..." if len(e.output) > 30 else e.output + else: + output = e.output + _logger.log( + loglvl, + "Failed to execute command '%s', return code: %s, output: '%s'", + cmd, + e.returncode, + output, + ) + + except Exception as e: + code = 99 + output = "{e}: {o}".format(e=e, o=output) + _logger.log( + loglvl, + "Failed to execute command '%s', return code: %s, output: '%s'", + cmd, + None, + output, + ) + + return output, code diff --git a/octoprint_netconnectd/util/pip_util.py b/octoprint_netconnectd/util/pip_util.py new file mode 100644 index 0000000..8991491 --- /dev/null +++ b/octoprint_netconnectd/util/pip_util.py @@ -0,0 +1,109 @@ +""" +copy pasta of octoprint_mrbeam/util/pip_util.py +only changed the logging to be mrbeam plugin independend +""" +import logging + +from octoprint.plugins.softwareupdate.updaters.pip import _get_pip_caller +from octoprint.util.pip import PipCaller +from cmd_exec import exec_cmd_output + +DISABLE_PIP_CHECK = "--disable-pip-version-check" +DISABLE_PY_WARNING = "--no-python-version-warning" + +# Dictionary of package versions available at different locations +# { +# /home/pi/oprint/bin/pip : { +# "OctoPrint x.x.x", +# ... +# }, +# /usr/share/iobeam/venv/bin/pip : { +# "iobeam y.y.y", +# ... +# } +# } +_pip_package_version_lists = {} + + +def get_version_of_pip_module(pip_name, pip_command=None, disable_pip_ver_check=True): + _logger = logging.getLogger(__name__ + ".get_version_of_pip_module") + global _pip_package_version_lists + version = None + returncode = -1 + if pip_command is None: + pip_command = "pip" + elif isinstance(pip_command, list): + pip_command = " ".join(pip_command) + # Checking for pip version outdate takes extra time and text output. + # NOTE: Older versions of pip do not have the --no-python-version-warning flag + for disabled in [ + DISABLE_PIP_CHECK, + ]: # DISABLE_PY_WARNING]: + if disable_pip_ver_check and not disabled in pip_command: + pip_command += " " + disabled + venv_packages = _pip_package_version_lists.get(pip_command, None) + + if venv_packages is None: + # perform a pip discovery and remember it for next time + command = "{pip_command} list".format(pip_command=pip_command) + _logger.debug("refreshing list of installed packages (%s list)", pip_command) + output, returncode = exec_cmd_output(command, shell=True, log=False) + if returncode == 0: + venv_packages = output.splitlines() + _pip_package_version_lists[pip_command] = venv_packages + elif returncode == 127: + _logger.error( + "`%s` was not found in local $PATH (returncode %s)", + pip_command, + returncode, + ) + return None + else: + _logger.warning("`%s list` returned code %s", pip_command, returncode) + return None + # Go through the package list available in our venv + for line in venv_packages: + token = line.split() + if len(token) >= 2 and token[0] == pip_name: + version = token[1] + break + _logger.debug("%s==%s", pip_name, version) + return version + + +def get_pip_caller(venv, _logger=None): + """ + gets the pip caller of the givenv venv + + Args: + venv: path to venv + _logger: logger to log call, stdout and stderr of the pip caller + + Returns: + PipCaller of the venv + """ + pip_caller = _get_pip_caller(command=venv) + if not isinstance(pip_caller, PipCaller): + raise RuntimeError("Can't run pip", None) + + def _log_call(*lines): + _log(lines, prefix=" ", stream="call") + + def _log_stdout(*lines): + _log(lines, prefix=">", stream="stdout") + + def _log_stderr(*lines): + _log(lines, prefix="!", stream="stderr") + + def _log(lines, prefix=None, stream=None, strip=True): + if strip: + lines = map(lambda x: x.strip(), lines) + for line in lines: + print(u"{} {}".format(prefix, line)) + + if _logger is not None: + pip_caller.on_log_call = _log_call + pip_caller.on_log_stdout = _log_stdout + pip_caller.on_log_stderr = _log_stderr + + return pip_caller From ee8fa6c1355a480a2bc9510d69dae8642784a7d3 Mon Sep 17 00:00:00 2001 From: Josef-MrBeam Date: Mon, 28 Mar 2022 10:34:44 +0200 Subject: [PATCH 11/11] add scripts to manifest --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index dad2e7d..ffa679b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,3 +4,4 @@ recursive-include octoprint_netconnectd/translations * include LICENSE include requirements.txt include README.md +graft octoprint_netconnectd/scripts \ No newline at end of file