diff --git a/.versioneer-lookup b/.versioneer-lookup index 9174f0ca6..41fd3a301 100644 --- a/.versioneer-lookup +++ b/.versioneer-lookup @@ -10,10 +10,10 @@ # master shall not use the lookup table, only tags master -# maintenance is currently the branch for preparation of maintenance release 1.2.6 +# maintenance is currently the branch for preparation of maintenance release 1.2.7 # so are any fix/... branches -maintenance 1.2.6-dev 96fc70bdb2dd74ba04c3071f70da385b0408904a -fix/.* 1.2.6-dev 96fc70bdb2dd74ba04c3071f70da385b0408904a +maintenance 1.2.7-dev 536bb31965db17b969e7c1c53e241ddac4ae1814 +fix/.* 1.2.7-dev 536bb31965db17b969e7c1c53e241ddac4ae1814 # Special case disconnected checkouts, e.g. 'git checkout ' \(detached.* diff --git a/CHANGELOG.md b/CHANGELOG.md index 541f25515..be40008fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,61 @@ # OctoPrint Changelog +## 1.2.7 (2015-10-20) + +### Improvements + + * [#1062](https://github.com/foosel/OctoPrint/issues/1062) - Plugin Manager + now has a configuration dialog that among other things allows defining the + used `pip` command if auto detection proves to be insufficient here. + * Allow defining additional `pip` parameters in Plugin Manager. That might + make `sudo`-less installation of plugins possible in situations where it's + tricky otherwise. + * Improved timelapse processing (backported from `devel` branch): + * Individually captured frames cannot "overtake" each other anymore through + usage of a capture queue. + * Notifications will now be shown when the capturing of the timelapse's + post roll happens, including an approximation of how long that will take. + * Usage of `requests` instead of `urllib` for fetching the snapshot, + appears to also have [positive effects on webcam compatibility](https://github.com/foosel/OctoPrint/issues/1078). + * Some more defensive escaping for various settings in the UI (e.g. webcam URL) + * Switch to more error resilient saving of configuration files and other files + modified during runtime (save to temporary file & move). Should reduce risk + of file corruption. + * Downloading GCODE and STL files should now set more fitting `Content-Type` + headers (`text/plain` and `application/sla`) for better client side support + for "Open with" like usage scenarios. + * Selecting z-triggered timelapse mode will now inform about that not working + when printing from SD card. + * Software Update Plugin: Removed "The web interface will now be reloaded" + notification after successful update since that became obsolete with + introduction of the "Reload Now" overlay. + * Updated required version of `psutil` and `netifaces` dependencies. + +### Bug Fixes + + * [#1057](https://github.com/foosel/OctoPrint/issues/1057) - Better error + resilience of the Software Update plugin against broken/incomplete update + configurations. + * [#1075](https://github.com/foosel/OctoPrint/issues/1075) - Fixed support + of `sudo` for installing plugins, but added big visible warning about it + as it's **not** recommended. + * [#1077](https://github.com/foosel/OctoPrint/issues/1077) - Do not hiccup + on [UTF-8 BOMs](https://en.wikipedia.org/wiki/Byte_order_mark) (or other + BOMs for that matter) at the beginning of GCODE files. + * Fixed an issue that caused user sessions to not be properly associated, + leading to Sessions getting duplicated, wrongly saved etc. + * Fixed internal server error (HTTP 500) response on REST API calls with + unset `Content-Type` header. + * Fixed an issue leading to drag-and-drop file uploads to trigger frontend + processing in various other file upload widgets. + * Fixed a documentation error. + * Fixed caching behaviour on GCODE/STL downloads, was setting the `ETag` + header improperly. + * Fixed GCODE viewer not properly detecting change of currently visualized + file on Windows systems. + +([Commits](https://github.com/foosel/OctoPrint/compare/1.2.6...1.2.7)) + ## 1.2.6 (2015-09-02) ### Improvements diff --git a/README.md b/README.md index 06d5b6b04..4d5a9ba0b 100644 --- a/README.md +++ b/README.md @@ -38,11 +38,25 @@ For information about how to go about contributions of any kind, please see the Installation ------------ -Installation instructions for installing from source for different operating systems can be found [on the wiki](https://github.com/foosel/OctoPrint/wiki#assorted-guides). +Installation instructions for installing from source for different operating +systems can be found [on the wiki](https://github.com/foosel/OctoPrint/wiki#assorted-guides). If you want to run OctoPrint on a Raspberry Pi you might want to take a look at [OctoPi](https://github.com/guysoft/OctoPi) which is a custom SD card image that includes OctoPrint plus dependencies. +The generic steps that should basically be done regardless of operating system +and runtime environment are the following (as *regular +user*, please keep your hands *off* of the `sudo` command here!) - this assumes +you already have Python 2.7, pip and virtualenv set up: + +1. Checkout OctoPrint: `git clone https://github.com/foosel/OctoPrint.git` +2. Change into the OctoPrint folder: `cd OctoPrint` +3. Create a user-owned virtual environment therein: `virtualenv --system-site-packages venv` +4. Install OctoPrint *into that virtual environment*: `./venv/bin/python setup.py install` + +You may then start the OctoPrint server via `/path/to/OctoPrint/venv/bin/octoprint`, see [Usage](#usage) +for details. + After installation, please make sure you follow the first-run wizard and set up access control as necessary. If you want to not only be notified about new releases but also be able to automatically upgrade to them from within diff --git a/docs/api/logs.rst b/docs/api/logs.rst index b2759aadb..875427842 100644 --- a/docs/api/logs.rst +++ b/docs/api/logs.rst @@ -15,7 +15,7 @@ Log file management Retrieve a list of available log files ====================================== -.. http:post:: /api/logs +.. http:get:: /api/logs Retrieve information regarding all log files currently available and regarding the disk space still available in the system on the location the log files are being stored. diff --git a/setup.py b/setup.py index 0fed3ef0c..80ed83c4b 100644 --- a/setup.py +++ b/setup.py @@ -26,13 +26,13 @@ "netaddr==0.7.17", "watchdog==0.8.3", "sarge==0.1.4", - "netifaces==0.8", + "netifaces==0.10", "pylru==1.0.9", "rsa==3.2", "pkginfo==1.2.1", "requests==2.7.0", "semantic_version==2.4.2", - "psutil==3.1.1" + "psutil==3.2.1" ] # Additional requirements for optional install options diff --git a/src/octoprint/events.py b/src/octoprint/events.py index 828862336..2cc765d9b 100644 --- a/src/octoprint/events.py +++ b/src/octoprint/events.py @@ -74,6 +74,9 @@ class Events(object): # Timelapse CAPTURE_START = "CaptureStart" CAPTURE_DONE = "CaptureDone" + CAPTURE_FAILED = "CaptureFailed" + POSTROLL_START = "PostRollStart" + POSTROLL_END = "PostRollEnd" MOVIE_RENDERING = "MovieRendering" MOVIE_DONE = "MovieDone" MOVIE_FAILED = "MovieFailed" diff --git a/src/octoprint/filemanager/__init__.py b/src/octoprint/filemanager/__init__.py index fad1fc41c..34f31680c 100644 --- a/src/octoprint/filemanager/__init__.py +++ b/src/octoprint/filemanager/__init__.py @@ -18,6 +18,11 @@ from .storage import LocalFileStorage from .util import AbstractFileWrapper, StreamWrapper, DiskFileWrapper +from collections import namedtuple + +ContentTypeMapping = namedtuple("ContentTypeMapping", "extensions, content_type") +ContentTypeDetector = namedtuple("ContentTypeDetector", "extensions, detector") + extensions = dict( ) @@ -25,11 +30,11 @@ def full_extension_tree(): result = dict( # extensions for 3d model files model=dict( - stl=["stl"] + stl=ContentTypeMapping(["stl"], "application/sla") ), # extensions for printable machine code machinecode=dict( - gcode=["gcode", "gco", "g"] + gcode=ContentTypeMapping(["gcode", "gco", "g"], "text/plain") ) ) @@ -68,8 +73,12 @@ def get_all_extensions(subtree=None): for key, value in subtree.items(): if isinstance(value, dict): result += get_all_extensions(value) + elif isinstance(value, (ContentTypeMapping, ContentTypeDetector)): + result += value.extensions elif isinstance(value, (list, tuple)): result += value + elif isinstance(subtree, (ContentTypeMapping, ContentTypeDetector)): + result = subtree.extensions elif isinstance(subtree, (list, tuple)): result = subtree return result @@ -79,7 +88,9 @@ def get_path_for_extension(extension, subtree=None): subtree = full_extension_tree() for key, value in subtree.items(): - if isinstance(value, (list, tuple)) and extension in value: + if isinstance(value, (ContentTypeMapping, ContentTypeDetector)) and extension in value.extensions: + return [key] + elif isinstance(value, (list, tuple)) and extension in value: return [key] elif isinstance(value, dict): path = get_path_for_extension(extension, subtree=value) @@ -88,6 +99,23 @@ def get_path_for_extension(extension, subtree=None): return None +def get_content_type_mapping_for_extension(extension, subtree=None): + if not subtree: + subtree = full_extension_tree() + + for key, value in subtree.items(): + content_extension_matches = isinstance(value, (ContentTypeMapping, ContentTypeDetector)) and extension in value. extensions + list_extension_matches = isinstance(value, (list, tuple)) and extension in value + + if content_extension_matches or list_extension_matches: + return value + elif isinstance(value, dict): + result = get_content_type_mapping_for_extension(extension, subtree=value) + if result is not None: + return result + + return None + def valid_extension(extension, type=None): if not type: return extension in get_all_extensions() @@ -106,6 +134,19 @@ def get_file_type(filename): extension = extension[1:].lower() return get_path_for_extension(extension) +def get_mime_type(filename): + _, extension = os.path.splitext(filename) + extension = extension[1:].lower() + mapping = get_content_type_mapping_for_extension(extension) + if mapping: + if isinstance(mapping, ContentTypeMapping) and mapping.content_type is not None: + return mapping.content_type + elif isinstance(mapping, ContentTypeDetector) and callable(mapping.detector): + result = mapping.detector(filename) + if result is not None: + return result + return "application/octet-stream" + class NoSuchStorage(Exception): pass diff --git a/src/octoprint/filemanager/util.py b/src/octoprint/filemanager/util.py index 105987f37..f3fe20fb3 100644 --- a/src/octoprint/filemanager/util.py +++ b/src/octoprint/filemanager/util.py @@ -7,6 +7,8 @@ import io +from octoprint.util import atomic_write + class AbstractFileWrapper(object): """ Wrapper for file representations to save to storages. @@ -85,7 +87,7 @@ def save(self, path): """ import shutil - with open(path, "wb") as dest: + with atomic_write(path, "wb") as dest: with self.stream() as source: shutil.copyfileobj(source, dest) diff --git a/src/octoprint/plugin/core.py b/src/octoprint/plugin/core.py index 913289de8..bcb28ce01 100644 --- a/src/octoprint/plugin/core.py +++ b/src/octoprint/plugin/core.py @@ -861,7 +861,7 @@ def initialize_implementations(self, additional_injects=None, additional_inject_ additional_pre_inits=additional_pre_inits, additional_post_inits=additional_post_inits) - self.logger.info("Initialized {count} plugin(s)".format(count=len(self.plugin_implementations))) + self.logger.info("Initialized {count} plugin implementation(s)".format(count=len(self.plugin_implementations))) def initialize_implementation_of_plugin(self, name, plugin, additional_injects=None, additional_inject_factories=None, additional_pre_inits=None, additional_post_inits=None): if plugin.implementation is None: @@ -955,15 +955,13 @@ def log_all_plugins(self, show_bundled=True, bundled_str=(" (bundled)", ""), sho self.logger.info("No plugins available") else: self.logger.info("{count} plugin(s) registered with the system:\n{plugins}".format(count=len(all_plugins), plugins="\n".join( - sorted( - map(lambda x: "| " + x.long_str(show_bundled=show_bundled, - bundled_strs=bundled_str, - show_location=show_location, - location_str=location_str, - show_enabled=show_enabled, - enabled_strs=enabled_str), - self.enabled_plugins.values()) - ) + map(lambda x: "| " + x.long_str(show_bundled=show_bundled, + bundled_strs=bundled_str, + show_location=show_location, + location_str=location_str, + show_enabled=show_enabled, + enabled_strs=enabled_str), + sorted(self.plugins.values(), key=lambda x: str(x).lower())) ))) def get_plugin(self, identifier, require_enabled=True): diff --git a/src/octoprint/plugins/cura/__init__.py b/src/octoprint/plugins/cura/__init__.py index 30de88741..fd326a109 100644 --- a/src/octoprint/plugins/cura/__init__.py +++ b/src/octoprint/plugins/cura/__init__.py @@ -393,7 +393,7 @@ def _load_profile(self, path): def _save_profile(self, path, profile, allow_overwrite=True): import yaml - with open(path, "wb") as f: + with octoprint.util.atomic_write(path, "wb") as f: yaml.safe_dump(profile, f, default_flow_style=False, indent=" ", allow_unicode=True) def _convert_to_engine(self, profile_path, printer_profile, posX, posY): diff --git a/src/octoprint/plugins/pluginmanager/__init__.py b/src/octoprint/plugins/pluginmanager/__init__.py index c7950f481..0c40562a1 100644 --- a/src/octoprint/plugins/pluginmanager/__init__.py +++ b/src/octoprint/plugins/pluginmanager/__init__.py @@ -82,14 +82,23 @@ def get_settings_defaults(self): repository="http://plugins.octoprint.org/plugins.json", repository_ttl=24*60, pip=None, + pip_args=None, dependency_links=False, hidden=[] ) def on_settings_save(self, data): + old_pip = self._settings.get(["pip"]) octoprint.plugin.SettingsPlugin.on_settings_save(self, data) + new_pip = self._settings.get(["pip"]) + self._repository_cache_ttl = self._settings.get_int(["repository_ttl"]) * 60 - self._pip_caller.refresh = True + if old_pip != new_pip: + self._pip_caller.configured = new_pip + try: + self._pip_caller.trigger_refresh() + except: + self._pip_caller ##~~ AssetPlugin @@ -169,7 +178,20 @@ def on_api_get(self, request): if "refresh_repository" in request.values and request.values["refresh_repository"] in valid_boolean_trues: self._repository_available = self._refresh_repository() - return jsonify(plugins=result, repository=dict(available=self._repository_available, plugins=self._repository_plugins), os=self._get_os(), octoprint=self._get_octoprint_version()) + return jsonify(plugins=result, + repository=dict( + available=self._repository_available, + plugins=self._repository_plugins + ), + os=self._get_os(), + octoprint=self._get_octoprint_version(), + pip=dict( + available=self._pip_caller.available, + command=self._pip_caller.command, + version=self._pip_caller.version_string, + use_sudo=self._pip_caller.use_sudo, + additional_args=self._settings.get(["pip_args"]) + )) def on_api_command(self, command, data): if not admin_permission.can(): @@ -428,6 +450,10 @@ def _call_pip(self, args): if self._pip_caller < self._pip_version_dependency_links: args.remove("--process-dependency-links") + additional_args = self._settings.get(["pip_args"]) + if additional_args: + args.append(additional_args) + return self._pip_caller.execute(*args) def _log_message(self, *lines): @@ -510,7 +536,7 @@ def _fetch_repository_from_url(self): try: import json - with open(self._repository_cache_path, "w+b") as f: + with octoprint.util.atomic_write(self._repository_cache_path, "wb") as f: json.dump(repo_data, f) except Exception as e: self._logger.exception("Error while saving repository data to {}: {}".format(self._repository_cache_path, str(e))) @@ -603,7 +629,8 @@ def _to_external_representation(self, plugin): pending_enable=(not plugin.enabled and plugin.key in self._pending_enable), pending_disable=(plugin.enabled and plugin.key in self._pending_disable), pending_install=(plugin.key in self._pending_install), - pending_uninstall=(plugin.key in self._pending_uninstall) + pending_uninstall=(plugin.key in self._pending_uninstall), + origin=plugin.origin.type ) __plugin_name__ = "Plugin Manager" diff --git a/src/octoprint/plugins/pluginmanager/static/js/pluginmanager.js b/src/octoprint/plugins/pluginmanager/static/js/pluginmanager.js index 7c5265ff5..75c8e2f39 100644 --- a/src/octoprint/plugins/pluginmanager/static/js/pluginmanager.js +++ b/src/octoprint/plugins/pluginmanager/static/js/pluginmanager.js @@ -6,6 +6,13 @@ $(function() { self.settingsViewModel = parameters[1]; self.printerState = parameters[2]; + self.config_repositoryUrl = ko.observable(); + self.config_repositoryTtl = ko.observable(); + self.config_pipCommand = ko.observable(); + self.config_pipAdditionalArgs = ko.observable(); + + self.configurationDialog = $("#settings_plugin_pluginmanager_configurationdialog"); + self.plugins = new ItemListHelper( "plugin.pluginmanager.installedplugins", { @@ -72,6 +79,12 @@ $(function() { self.followDependencyLinks = ko.observable(false); + self.pipAvailable = ko.observable(false); + self.pipCommand = ko.observable(); + self.pipVersion = ko.observable(); + self.pipUseSudo = ko.observable(); + self.pipAdditionalArgs = ko.observable(); + self.working = ko.observable(false); self.workingTitle = ko.observable(); self.workingDialog = undefined; @@ -86,11 +99,15 @@ $(function() { }; self.enableUninstall = function(data) { - return self.enableManagement() && !data.bundled && data.key != 'pluginmanager' && !data.pending_uninstall; + return self.enableManagement() + && (data.origin != "entry_point" || self.pipAvailable()) + && !data.bundled + && data.key != 'pluginmanager' + && !data.pending_uninstall; }; self.enableRepoInstall = function(data) { - return self.enableManagement() && self.isCompatible(data); + return self.enableManagement() && self.pipAvailable() && self.isCompatible(data); }; self.invalidUrl = ko.computed(function() { @@ -100,7 +117,7 @@ $(function() { self.enableUrlInstall = ko.computed(function() { var url = self.installUrl(); - return self.enableManagement() && url !== undefined && url.trim() != "" && !self.invalidUrl(); + return self.enableManagement() && self.pipAvailable() && url !== undefined && url.trim() != "" && !self.invalidUrl(); }); self.invalidArchive = ko.computed(function() { @@ -110,7 +127,7 @@ $(function() { self.enableArchiveInstall = ko.computed(function() { var name = self.uploadFilename(); - return self.enableManagement() && name !== undefined && name.trim() != "" && !self.invalidArchive(); + return self.enableManagement() && self.pipAvailable() && name !== undefined && name.trim() != "" && !self.invalidArchive(); }); self.uploadElement.fileupload({ @@ -167,7 +184,8 @@ $(function() { self.fromResponse = function(data) { self._fromPluginsResponse(data.plugins); - self._fromRepositoryResponse(data.repository) + self._fromRepositoryResponse(data.repository); + self._fromPipResponse(data.pip); }; self._fromPluginsResponse = function(data) { @@ -188,6 +206,21 @@ $(function() { } }; + self._fromPipResponse = function(data) { + self.pipAvailable(data.available); + if (data.available) { + self.pipCommand(data.command); + self.pipVersion(data.version); + self.pipUseSudo(data.use_sudo); + self.pipAdditionalArgs(data.additional_args); + } else { + self.pipCommand(undefined); + self.pipVersion(undefined); + self.pipUseSudo(undefined); + self.pipAdditionalArgs(undefined); + } + }; + self.requestData = function(includeRepo) { if (!self.loginState.isAdmin()) { return; @@ -343,6 +376,58 @@ $(function() { self.requestData(true); }; + self.showPluginSettings = function() { + self._copyConfig(); + self.configurationDialog.modal(); + }; + + self.savePluginSettings = function() { + var pipCommand = self.config_pipCommand(); + if (pipCommand != undefined && pipCommand.trim() == "") { + pipCommand = null; + } + + var repository = self.config_repositoryUrl(); + if (repository != undefined && repository.trim() == "") { + repository = null; + } + + var repositoryTtl; + try { + repositoryTtl = parseInt(self.config_repositoryTtl()); + } catch (ex) { + repositoryTtl = null; + } + + var pipArgs = self.config_pipAdditionalArgs(); + if (pipArgs != undefined && pipArgs.trim() == "") { + pipArgs = null; + } + + var data = { + plugins: { + pluginmanager: { + repository: repository, + repository_ttl: repositoryTtl, + pip: pipCommand, + pip_args: pipArgs + } + } + }; + self.settingsViewModel.saveData(data, function() { + self.configurationDialog.modal("hide"); + self._copyConfig(); + self.refreshRepository(); + }); + }; + + self._copyConfig = function() { + self.config_repositoryUrl(self.settingsViewModel.settings.plugins.pluginmanager.repository()); + self.config_repositoryTtl(self.settingsViewModel.settings.plugins.pluginmanager.repository_ttl()); + self.config_pipCommand(self.settingsViewModel.settings.plugins.pluginmanager.pip()); + self.config_pipAdditionalArgs(self.settingsViewModel.settings.plugins.pluginmanager.pip_args()); + }; + self.installed = function(data) { return _.includes(self.installedPlugins(), data.id); }; diff --git a/src/octoprint/plugins/pluginmanager/templates/pluginmanager_settings.jinja2 b/src/octoprint/plugins/pluginmanager/templates/pluginmanager_settings.jinja2 index b855dc81f..d399b593c 100644 --- a/src/octoprint/plugins/pluginmanager/templates/pluginmanager_settings.jinja2 +++ b/src/octoprint/plugins/pluginmanager/templates/pluginmanager_settings.jinja2 @@ -4,7 +4,31 @@ {% endmacro %} +{% macro pluginmanager_nopip() %} +
{% trans %} + The pip command could not be found. + Please configure it manually. No installation and uninstallation of plugin + packages is possible while pip is unavailable. +{% endtrans %}
+{% endmacro %} + +{% macro pluginmanager_sudopip() %} +
{% trans %} + The pip command is configured to use sudo. This + is not recommended due to security reasons. It is strongly + suggested you install OctoPrint under a + user-owned virtual environment + so that the use of sudo is not needed for plugin management. +{% endtrans %}
+{% endmacro %} + {{ pluginmanager_printing() }} +{{ pluginmanager_nopip() }} +{{ pluginmanager_sudopip() }} + +
+ +

{{ _('Installed Plugins') }}

@@ -47,6 +71,10 @@ +

+ Using pip at "" (Version , additional arguments: ) +

+ + + + diff --git a/src/octoprint/plugins/softwareupdate/__init__.py b/src/octoprint/plugins/softwareupdate/__init__.py index ad1683a1d..a3f7d2ef7 100644 --- a/src/octoprint/plugins/softwareupdate/__init__.py +++ b/src/octoprint/plugins/softwareupdate/__init__.py @@ -59,6 +59,7 @@ def _get_configured_checks(self): self._refresh_configured_checks = False self._configured_checks = self._settings.get(["checks"], merged=True) update_check_hooks = self._plugin_manager.get_hooks("octoprint.plugin.softwareupdate.check_config") + check_providers = self._settings.get(["check_providers"], merged=True) for name, hook in update_check_hooks.items(): try: hook_checks = hook() @@ -66,9 +67,23 @@ def _get_configured_checks(self): self._logger.exception("Error while retrieving update information from plugin {name}".format(**locals())) else: for key, data in hook_checks.items(): + check_providers[key] = name if key in self._configured_checks: data = dict_merge(data, self._configured_checks[key]) self._configured_checks[key] = data + self._settings.set(["check_providers"], check_providers) + self._settings.save() + + # we only want to process checks that came from plugins for + # which the plugins are still installed and enabled + config_checks = self._settings.get(["checks"]) + plugin_and_not_enabled = lambda k: k in check_providers and \ + not check_providers[k] in self._plugin_manager.enabled_plugins + obsolete_plugin_checks = filter(plugin_and_not_enabled, + config_checks.keys()) + for key in obsolete_plugin_checks: + self._logger.debug("Check for key {} was provided by plugin {} that's no longer available, ignoring it".format(key, check_providers[key])) + del self._configured_checks[key] return self._configured_checks @@ -134,6 +149,7 @@ def get_settings_defaults(self): }, }, "pip_command": None, + "check_providers": {}, "cache_ttl": 24 * 60, } @@ -400,14 +416,13 @@ def get_current_versions(self, check_targets=None, force=False): if not target in check_targets: continue - populated_check = self._populated_check(target, check) - try: + populated_check = self._populated_check(target, check) target_information, target_update_available, target_update_possible = self._get_current_version(target, populated_check, force=force) if target_information is None: target_information = dict() except exceptions.UnknownCheckType: - self._logger.warn("Unknown update check type for %s" % target) + self._logger.warn("Unknown update check type for target {}: {}".format(target, check.get("type", ""))) continue target_information = dict_merge(dict(local=dict(name="unknown", value="unknown"), remote=dict(name="unknown", value="unknown")), target_information) @@ -655,6 +670,9 @@ def _perform_restart(self, restart_command): raise exceptions.RestartFailed() def _populated_check(self, target, check): + if not "type" in check: + raise exceptions.UnknownCheckType() + result = dict(check) if target == "octoprint": @@ -687,13 +705,6 @@ def _get_version_checker(self, target, check): if not "type" in check: raise exceptions.ConfigurationInvalid("no check type defined") - if target == "octoprint": - from octoprint._version import get_versions - from flask.ext.babel import gettext - check["displayName"] = gettext("OctoPrint") - check["displayVersion"] = "{octoprint_version}" - check["current"] = get_versions()["version"] - check_type = check["type"] if check_type == "github_release": return version_checks.github_release diff --git a/src/octoprint/plugins/softwareupdate/static/js/softwareupdate.js b/src/octoprint/plugins/softwareupdate/static/js/softwareupdate.js index fd4ba14ff..8b75047b4 100644 --- a/src/octoprint/plugins/softwareupdate/static/js/softwareupdate.js +++ b/src/octoprint/plugins/softwareupdate/static/js/softwareupdate.js @@ -27,6 +27,8 @@ $(function() { {"key": "git_commit", "name": gettext("Commit")} ]; + self.reloadOverlay = $("#reloadui_overlay"); + self.versions = new ItemListHelper( "plugin.softwareupdate.versions", { @@ -329,18 +331,10 @@ $(function() { self.onDataUpdaterReconnect = function() { if (self.waitingForRestart) { self.waitingForRestart = false; - - var options = { - title: gettext("Restart successful!"), - text: gettext("The server was restarted successfully. The page will now reload automatically."), - type: "success", - hide: false - }; - self._showPopup(options); self.updateInProgress = false; - - var delay = 5 + Math.floor(Math.random() * 5) + 1; - setTimeout(function() {location.reload(true);}, delay * 1000); + if (!self.reloadOverlay.is(":visible")) { + self.reloadOverlay.show(); + } } }; diff --git a/src/octoprint/printer/standard.py b/src/octoprint/printer/standard.py index c332182c5..8e11b6421 100644 --- a/src/octoprint/printer/standard.py +++ b/src/octoprint/printer/standard.py @@ -680,7 +680,7 @@ def _setJobData(self, filename, filesize, sd): # Use a string for mtime because it could be float and the # javascript needs to exact match if not sd: - date = int(os.stat(path_on_disk).st_ctime) + date = int(os.stat(path_on_disk).st_mtime) try: fileData = self._fileManager.get_metadata(FileDestinations.SDCARD if sd else FileDestinations.LOCAL, path_on_disk) diff --git a/src/octoprint/server/__init__.py b/src/octoprint/server/__init__.py index 18ea638df..f76f9cb8c 100644 --- a/src/octoprint/server/__init__.py +++ b/src/octoprint/server/__init__.py @@ -81,7 +81,7 @@ def on_identity_loaded(sender, identity): if user is None: return - identity.provides.add(UserNeed(user.get_name())) + identity.provides.add(UserNeed(user.get_id())) if user.is_user(): identity.provides.add(RoleNeed("user")) if user.is_admin(): @@ -98,9 +98,9 @@ def load_user(id): if userManager is not None: if sessionid: - return userManager.findUser(username=id, session=sessionid) + return userManager.findUser(userid=id, session=sessionid) else: - return userManager.findUser(username=id) + return userManager.findUser(userid=id) return users.DummyUser() @@ -330,13 +330,34 @@ def template_disabled(name, plugin): upload_suffixes = dict(name=s.get(["server", "uploads", "nameSuffix"]), path=s.get(["server", "uploads", "pathSuffix"])) + def mime_type_guesser(path): + from octoprint.filemanager import get_mime_type + return get_mime_type(path) + + download_handler_kwargs = dict( + as_attachment=True, + allow_client_caching=False + ) + additional_mime_types=dict(mime_type_guesser=mime_type_guesser) + admin_validator = dict(access_validation=util.tornado.access_validation_factory(app, loginManager, util.flask.user_validator)) + no_hidden_files_validator = dict(path_validation=util.tornado.path_validation_factory(lambda path: not os.path.basename(path).startswith("."), status_code=404)) + + def joined_dict(*dicts): + if not len(dicts): + return dict() + + joined = dict() + for d in dicts: + joined.update(d) + return joined + server_routes = self._router.urls + [ # various downloads - (r"/downloads/timelapse/([^/]*\.mpg)", util.tornado.LargeResponseHandler, dict(path=s.getBaseFolder("timelapse"), as_attachment=True)), - (r"/downloads/files/local/(.*)", util.tornado.LargeResponseHandler, dict(path=s.getBaseFolder("uploads"), as_attachment=True, path_validation=util.tornado.path_validation_factory(lambda path: not os.path.basename(path).startswith("."), status_code=404))), - (r"/downloads/logs/([^/]*)", util.tornado.LargeResponseHandler, dict(path=s.getBaseFolder("logs"), as_attachment=True, access_validation=util.tornado.access_validation_factory(app, loginManager, util.flask.admin_validator))), + (r"/downloads/timelapse/([^/]*\.mpg)", util.tornado.LargeResponseHandler, joined_dict(dict(path=s.getBaseFolder("timelapse")), download_handler_kwargs, no_hidden_files_validator)), + (r"/downloads/files/local/(.*)", util.tornado.LargeResponseHandler, joined_dict(dict(path=s.getBaseFolder("uploads")), download_handler_kwargs, no_hidden_files_validator, additional_mime_types)), + (r"/downloads/logs/([^/]*)", util.tornado.LargeResponseHandler, joined_dict(dict(path=s.getBaseFolder("logs")), download_handler_kwargs, admin_validator)), # camera snapshot - (r"/downloads/camera/current", util.tornado.UrlForwardHandler, dict(url=s.get(["webcam", "snapshot"]), as_attachment=True, access_validation=util.tornado.access_validation_factory(app, loginManager, util.flask.user_validator))), + (r"/downloads/camera/current", util.tornado.UrlProxyHandler, dict(url=s.get(["webcam", "snapshot"]), as_attachment=True, access_validation=util.tornado.access_validation_factory(app, loginManager, util.flask.user_validator))), # generated webassets (r"/static/webassets/(.*)", util.tornado.LargeResponseHandler, dict(path=os.path.join(s.getBaseFolder("generated"), "webassets"))) ] diff --git a/src/octoprint/server/api/timelapse.py b/src/octoprint/server/api/timelapse.py index bef855365..63b5be342 100644 --- a/src/octoprint/server/api/timelapse.py +++ b/src/octoprint/server/api/timelapse.py @@ -29,14 +29,14 @@ def getTimelapseData(): config = {"type": "off"} if timelapse is not None and isinstance(timelapse, octoprint.timelapse.ZTimelapse): config["type"] = "zchange" - config["postRoll"] = timelapse.postRoll() - config["fps"] = timelapse.fps() + config["postRoll"] = timelapse.post_roll + config["fps"] = timelapse.fps elif timelapse is not None and isinstance(timelapse, octoprint.timelapse.TimedTimelapse): config["type"] = "timed" - config["postRoll"] = timelapse.postRoll() - config["fps"] = timelapse.fps() + config["postRoll"] = timelapse.post_roll + config["fps"] = timelapse.fps config.update({ - "interval": timelapse.interval() + "interval": timelapse.interval }) files = octoprint.timelapse.getFinishedTimelapses() diff --git a/src/octoprint/server/util/flask.py b/src/octoprint/server/util/flask.py index 30136222e..7fe8ae34a 100644 --- a/src/octoprint/server/util/flask.py +++ b/src/octoprint/server/util/flask.py @@ -227,8 +227,10 @@ def passive_login(): user = flask.ext.login.current_user if user is not None and not user.is_anonymous(): - flask.g.user = user flask.ext.principal.identity_changed.send(flask.current_app._get_current_object(), identity=flask.ext.principal.Identity(user.get_id())) + if hasattr(user, "get_session"): + flask.session["usersession.id"] = user.get_session() + flask.g.user = user return flask.jsonify(user.asDict()) elif settings().getBoolean(["accessControl", "autologinLocal"]) \ and settings().get(["accessControl", "autologinAs"]) is not None \ @@ -252,7 +254,7 @@ def passive_login(): logger = logging.getLogger(__name__) logger.exception("Could not autologin user %s for networks %r" % (autologinAs, localNetworks)) - return ("", 204) + return "", 204 #~~ cache decorator for cacheable views @@ -528,7 +530,8 @@ def get_remote_address(request): def get_json_command_from_request(request, valid_commands): - if not "application/json" in request.headers["Content-Type"]: + content_type = request.headers.get("Content-Type", None) + if content_type is None or not "application/json" in content_type: return None, None, make_response("Expected content-type JSON", 400) data = request.json diff --git a/src/octoprint/server/util/tornado.py b/src/octoprint/server/util/tornado.py index ce1bdd002..30753f34f 100644 --- a/src/octoprint/server/util/tornado.py +++ b/src/octoprint/server/util/tornado.py @@ -768,6 +768,8 @@ class LargeResponseHandler(tornado.web.StaticFileHandler): :class:``~tornado.web.StaticFileHandler`` as the ``default_filename`` keyword parameter). Defaults to ``None``. as_attachment (bool): Whether to serve requested files with ``Content-Disposition: attachment`` header (``True``) or not. Defaults to ``False``. + allow_client_caching (bool): Whether to allow the client to cache (by not setting any ``Cache-Control`` or + ``Expires`` headers on the response) or not. access_validation (function): Callback to call in the ``get`` method to validate access to the resource. Will be called with ``self.request`` as parameter which contains the full tornado request object. Should raise a ``tornado.web.HTTPError`` if access is not allowed in which case the request will not be further processed. @@ -776,13 +778,22 @@ class LargeResponseHandler(tornado.web.StaticFileHandler): with the requested path as parameter. Should raise a ``tornado.web.HTTPError`` (e.g. an 404) if the requested path does not pass validation in which case the request will not be further processed. Defaults to ``None`` and hence no path validation being performed. + etag_generator (function): Callback to call for generating the value of the ETag response header. Will be + called with the response handler as parameter. May return ``None`` to prevent the ETag response header + from being set. If not provided the last modified time of the file in question will be used as returned + by ``get_content_version``. """ - def initialize(self, path, default_filename=None, as_attachment=False, access_validation=None, path_validation=None): + def initialize(self, path, default_filename=None, as_attachment=False, allow_client_caching=True, + access_validation=None, path_validation=None, etag_generator=None, + mime_type_guesser=None): tornado.web.StaticFileHandler.initialize(self, os.path.abspath(path), default_filename) self._as_attachment = as_attachment + self._allow_client_caching = allow_client_caching self._access_validation = access_validation self._path_validation = path_validation + self._etag_generator = etag_generator + self._mime_type_guesser = mime_type_guesser def get(self, path, include_body=True): if self._access_validation is not None: @@ -796,6 +807,24 @@ def set_extra_headers(self, path): if self._as_attachment: self.set_header("Content-Disposition", "attachment") + if not self._allow_client_caching: + self.set_header("Cache-Control", "max-age=0, must-revalidate, private") + self.set_header("Expires", "-1") + + def compute_etag(self): + if self._etag_generator is not None: + return self._etag_generator(self) + else: + return self.get_content_version(self.absolute_path) + + def get_content_type(self): + if self._mime_type_guesser is not None: + type = self._mime_type_guesser(self.absolute_path) + if type is not None: + return type + + return tornado.web.StaticFileHandler.get_content_type(self) + @classmethod def get_content_version(cls, abspath): import os @@ -805,7 +834,7 @@ def get_content_version(cls, abspath): ##~~ URL Forward Handler for forwarding requests to a preconfigured static URL -class UrlForwardHandler(tornado.web.RequestHandler): +class UrlProxyHandler(tornado.web.RequestHandler): """ `tornado.web.RequestHandler `_ that proxies requests to a preconfigured url and returns the response. Allows delivery of the requested content as attachment @@ -815,7 +844,7 @@ class UrlForwardHandler(tornado.web.RequestHandler): for making the request to the configured endpoint and return the body of the client response with the status code from the client response and the following headers: - * ``Date``, ``Cache-Control``, ``Server``, ``Content-Type`` and ``Location`` will be copied over. + * ``Date``, ``Cache-Control``, ``Expires``, ``ETag``, ``Server``, ``Content-Type`` and ``Location`` will be copied over. * If ``as_attachment`` is set to True, ``Content-Disposition`` will be set to ``attachment``. If ``basename`` is set including the attachement's ``filename`` attribute will be set to the base name followed by the extension guessed based on the MIME type from the ``Content-Type`` header of the response. If no extension can be guessed @@ -865,7 +894,7 @@ def handle_response(self, response): filename = None self.set_status(response.code) - for name in ("Date", "Cache-Control", "Server", "Content-Type", "Location"): + for name in ("Date", "Cache-Control", "Server", "Content-Type", "Location", "Expires", "ETag"): value = response.headers.get(name) if value: self.set_header(name, value) diff --git a/src/octoprint/server/views.py b/src/octoprint/server/views.py index 905e14366..f89bd9dad 100644 --- a/src/octoprint/server/views.py +++ b/src/octoprint/server/views.py @@ -16,11 +16,16 @@ debug, LOCALES, VERSION, DISPLAY_VERSION, UI_API_KEY, BRANCH from octoprint.settings import settings +import re + from . import util import logging _logger = logging.getLogger(__name__) +_valid_id_re = re.compile("[a-z_]+") +_valid_div_re = re.compile("[a-zA-Z_-]+") + @app.route("/") @util.flask.cached(refreshif=lambda: util.flask.cache_check_headers() or "_refresh" in request.values, key=lambda: "view/%s/%s" % (request.path, g.locale), @@ -367,6 +372,9 @@ def _process_template_config(name, implementation, rule, config=None, counter=1) data["_div"] = rule["div"](name) if "suffix" in data: data["_div"] = data["_div"] + data["suffix"] + if not _valid_div_re.match(data["_div"]): + _logger.warn("Template config {} contains invalid div identifier {}, skipping it".format(name, data["_div"])) + return None if not "template" in data: data["template"] = rule["template"](name) @@ -378,6 +386,7 @@ def _process_template_config(name, implementation, rule, config=None, counter=1) data_bind = "allowBindings: true" if "data_bind" in data: data_bind = data_bind + ", " + data["data_bind"] + data_bind = data_bind.replace("\"", "\\\"") data["data_bind"] = data_bind data["_key"] = "plugin_" + name diff --git a/src/octoprint/settings.py b/src/octoprint/settings.py index fa3f0e5fb..733d24b8d 100644 --- a/src/octoprint/settings.py +++ b/src/octoprint/settings.py @@ -30,6 +30,8 @@ import re import uuid +from octoprint.util import atomic_write + _APPNAME = "OctoPrint" _instance = None @@ -1057,7 +1059,7 @@ def saveScript(self, script_type, name, script): path, _ = os.path.split(filename) if not os.path.exists(path): os.makedirs(path) - with open(filename, "w+") as f: + with atomic_write(filename, "wb") as f: f.write(script) def _default_basedir(applicationName): diff --git a/src/octoprint/static/js/app/dataupdater.js b/src/octoprint/static/js/app/dataupdater.js index 53f5ef705..26bd5f7e5 100644 --- a/src/octoprint/static/js/app/dataupdater.js +++ b/src/octoprint/static/js/app/dataupdater.js @@ -177,6 +177,7 @@ function DataUpdater(allViewModels) { var type = data["type"]; var payload = data["payload"]; var html = ""; + var format = {}; log.debug("Got event " + type + " with payload: " + JSON.stringify(payload)); @@ -187,7 +188,31 @@ function DataUpdater(allViewModels) { } else if (type == "MovieFailed") { html = "

" + _.sprintf(gettext("Rendering of timelapse %(movie_basename)s failed with return code %(returncode)s"), payload) + "

"; html += pnotifyAdditionalInfo('
' + payload.error + '
'); - new PNotify({title: gettext("Rendering failed"), text: html, type: "error", hide: false}); + new PNotify({ + title: gettext("Rendering failed"), + text: html, + type: "error", + hide: false + }); + } else if (type == "PostRollStart") { + var title = gettext("Capturing timelapse postroll"); + + var text; + if (!payload.postroll_duration) { + text = _.sprintf(gettext("Now capturing timelapse post roll, this will take only a moment..."), format); + } else { + if (payload.postroll_duration > 60) { + format = {duration: _.sprintf(gettext("%(minutes)d min"), {minutes: payload.postroll_duration / 60})}; + } else { + format = {duration: _.sprintf(gettext("%(seconds)d sec"), {seconds: payload.postroll_duration})}; + } + text = _.sprintf(gettext("Now capturing timelapse post roll, this will take approximately %(duration)s..."), format); + } + + new PNotify({ + title: title, + text: text + }); } else if (type == "SlicingStarted") { gcodeUploadProgress.addClass("progress-striped").addClass("active"); gcodeUploadProgressBar.css("width", "100%"); diff --git a/src/octoprint/static/js/app/main.js b/src/octoprint/static/js/app/main.js index 56617dbae..bab804bb9 100644 --- a/src/octoprint/static/js/app/main.js +++ b/src/octoprint/static/js/app/main.js @@ -21,6 +21,15 @@ $(function() { headers: {"X-Api-Key": UI_API_KEY} }); + //~~ Initialize file upload plugin + + $.widget("blueimp.fileupload", $.blueimp.fileupload, { + options: { + dropZone: null, + pasteZone: null + } + }); + //~~ Initialize i18n var catalog = window["BABEL_TO_LOAD_" + LOCALE]; diff --git a/src/octoprint/templates/dialogs/settings.jinja2 b/src/octoprint/templates/dialogs/settings.jinja2 index f3d3f708c..a87fa6a6a 100644 --- a/src/octoprint/templates/dialogs/settings.jinja2 +++ b/src/octoprint/templates/dialogs/settings.jinja2 @@ -20,7 +20,7 @@ class="{% if mark_active %}active{% set mark_active = False %}{% endif %} {% if "classes_link" in data %}{{ data.classes_link|join(' ') }}{% elif "classes" in data %}{{ data.classes|join(' ') }}{% endif %}" {% if "styles_link" in data %} style="{{ data.styles_link|join(', ') }}" {% elif "styles" in data %} style="{{ data.styles|join(', ') }}" {% endif %} > - {{ entry }} + {{ entry|e }} {% if "custom_bindings" not in data or data["custom_bindings"] %}{% endif %} {% endif %} diff --git a/src/octoprint/templates/dialogs/usersettings.jinja2 b/src/octoprint/templates/dialogs/usersettings.jinja2 index 00cbcc6c2..be405c153 100644 --- a/src/octoprint/templates/dialogs/usersettings.jinja2 +++ b/src/octoprint/templates/dialogs/usersettings.jinja2 @@ -17,7 +17,7 @@ class="{% if mark_active %}active{% set mark_active = False %}{% endif %} {% if "classes_link" in data %}{{ data.classes_link|join(' ') }}{% elif "classes" in data %}{{ data.classes|join(' ') }}{% endif %}" {% if "styles_link" in data %} style="{{ data.styles_link|join(', ') }}" {% elif "styles" in data %} style="{{ data.styles|join(', ') }}" {% endif %} > - {{ entry }} + {{ entry|e }} {% if "custom_bindings" not in data or data["custom_bindings"] %}{% endif %} {% endif %} diff --git a/src/octoprint/templates/index.jinja2 b/src/octoprint/templates/index.jinja2 index 1a58a82bf..a5de49c1f 100644 --- a/src/octoprint/templates/index.jinja2 +++ b/src/octoprint/templates/index.jinja2 @@ -55,7 +55,7 @@ >
- {% if "icon" in data %} {% endif %}{{ entry }} + {% if "icon" in data %} {% endif %}{{ entry|e }} {% if "template_header" in data %} {% include data.template_header ignore missing %} @@ -87,7 +87,7 @@ {% if "data_bind" in data %}data-bind="{{ data.data_bind }}"{% endif %} {% if "styles_link" in data %} style="{{ data.styles_link|join(', ') }}" {% elif "styles" in data %} style="{{ data.styles|join(', ') }}" {% endif %} > - {{ entry }} + {{ entry|e }} {% if "custom_bindings" not in data or data["custom_bindings"] %}{% endif %} {% endfor %} @@ -112,7 +112,7 @@
    -
  • {{ _('Version') }}: {{ display_version }}
    • +
  • {{ _('Version') }}: {{ display_version|e }}
  • {{ _('Homepage') }}
    • diff --git a/src/octoprint/templates/initscript.jinja2 b/src/octoprint/templates/initscript.jinja2 index c5a39dc14..8d212c621 100644 --- a/src/octoprint/templates/initscript.jinja2 +++ b/src/octoprint/templates/initscript.jinja2 @@ -10,24 +10,24 @@ var CONFIG_TIMELAPSEFILESPERPAGE = 10; var CONFIG_LOGFILESPERPAGE = 10; var CONFIG_USERSPERPAGE = 10; - var CONFIG_WEBCAM_STREAM = "{{ webcamStream }}"; + var CONFIG_WEBCAM_STREAM = "{{ webcamStream|e }}"; var CONFIG_ACCESS_CONTROL = {% if enableAccessControl -%} true; {% else %} false; {%- endif %} var CONFIG_SD_SUPPORT = {% if enableSdSupport -%} true; {% else %} false; {%- endif %} var CONFIG_FIRST_RUN = {% if firstRun -%} true; {% else %} false; {%- endif %} var CONFIG_TEMPERATURE_GRAPH = {% if enableTemperatureGraph -%} true; {% else %} false; {%- endif %} - var CONFIG_GCODE_SIZE_THRESHOLD = {{ gcodeThreshold }}; - var CONFIG_GCODE_MOBILE_SIZE_THRESHOLD = {{ gcodeMobileThreshold }}; + var CONFIG_GCODE_SIZE_THRESHOLD = {{ gcodeThreshold|e }}; + var CONFIG_GCODE_MOBILE_SIZE_THRESHOLD = {{ gcodeMobileThreshold|e }}; var SOCKJS_URI = "{{ url_for('index') }}" + "sockjs"; var SOCKJS_DEBUG = CONFIG_DEBUG; // sockjs should define CLOSE_NORMAL for us, but they don't (from ws spec) var SOCKJS_CLOSE_NORMAL = 1000; - var UI_API_KEY = "{{ uiApiKey }}"; - var VERSION = "{{ version }}"; - var DISPLAY_VERSION = "{{ display_version }}"; - var BRANCH = "{{ branch }}"; - var LOCALE = "{{ g.locale }}"; + var UI_API_KEY = "{{ uiApiKey|e }}"; + var VERSION = "{{ version|e }}"; + var DISPLAY_VERSION = "{{ display_version|e }}"; + var BRANCH = "{{ branch|e }}"; + var LOCALE = "{{ g.locale|e }}"; var AVAILABLE_LOCALES = {{ locales|tojson }}; var OCTOPRINT_VIEWMODELS = []; diff --git a/src/octoprint/templates/tabs/timelapse.jinja2 b/src/octoprint/templates/tabs/timelapse.jinja2 index 5fda40a9f..acea78ade 100644 --- a/src/octoprint/templates/tabs/timelapse.jinja2 +++ b/src/octoprint/templates/tabs/timelapse.jinja2 @@ -7,10 +7,11 @@ {{ _('Warning') }} {{ _('Do not use with spiralized ("Joris") vases or similar continuous Z models.') }} + {{ _('Note') }} {% trans %}Does not work when printing from the printer's SD Card (no way to detect the change in Z reliably). Use "Timed" mode for those prints instead.{% endtrans %}
    diff --git a/src/octoprint/timelapse.py b/src/octoprint/timelapse.py index e1e0b87f9..9ce2e3fcc 100644 --- a/src/octoprint/timelapse.py +++ b/src/octoprint/timelapse.py @@ -6,13 +6,13 @@ import logging import os import threading -import urllib import time -import subprocess import fnmatch import datetime import sys import shutil +import Queue +import requests import octoprint.util as util @@ -58,7 +58,7 @@ def notifyCallbacks(timelapse): if timelapse is None: config = None else: - config = timelapse.configData() + config = timelapse.config_data() for callback in updateCallbacks: try: callback.sendTimelapseConfig(config) except: logging.getLogger(__name__).exception("Exception while pushing timelapse configuration") @@ -86,12 +86,12 @@ def configureTimelapse(config=None, persist=False): if type is None or "off" == type: current = None elif "zchange" == type: - current = ZTimelapse(postRoll=postRoll, fps=fps) + current = ZTimelapse(post_roll=postRoll, fps=fps) elif "timed" == type: interval = 10 if "options" in config and "interval" in config["options"] and config["options"]["interval"] > 0: interval = config["options"]["interval"] - current = TimedTimelapse(postRoll=postRoll, interval=interval, fps=fps) + current = TimedTimelapse(post_roll=postRoll, interval=interval, fps=fps) notifyCallbacks(current) @@ -101,72 +101,83 @@ def configureTimelapse(config=None, persist=False): class Timelapse(object): - def __init__(self, postRoll=0, fps=25): + QUEUE_ENTRY_TYPE_CAPTURE = "capture" + QUEUE_ENTRY_TYPE_CALLBACK = "callback" + + def __init__(self, post_roll=0, fps=25): self._logger = logging.getLogger(__name__) - self._imageNumber = None - self._inTimelapse = False - self._gcodeFile = None + self._image_number = None + self._in_timelapse = False + self._gcode_file = None - self._postRoll = postRoll - self._postRollStart = None - self._onPostRollDone = None + self._post_roll = post_roll + self._on_post_roll_done = None - self._captureDir = settings().getBaseFolder("timelapse_tmp") - self._movieDir = settings().getBaseFolder("timelapse") - self._snapshotUrl = settings().get(["webcam", "snapshot"]) - self._ffmpegThreads = settings().get(["webcam", "ffmpegThreads"]) + self._capture_dir = settings().getBaseFolder("timelapse_tmp") + self._movie_dir = settings().getBaseFolder("timelapse") + self._snapshot_url = settings().get(["webcam", "snapshot"]) + self._ffmpeg_threads = settings().get(["webcam", "ffmpegThreads"]) self._fps = fps - self._renderThread = None - self._captureMutex = threading.Lock() + self._render_thread = None + + self._capture_mutex = threading.Lock() + self._capture_queue = Queue.Queue() + self._capture_queue_active = True + + self._capture_queue_thread = threading.Thread(target=self._capture_queue_worker) + self._capture_queue_thread.daemon = True + self._capture_queue_thread.start() # subscribe events - eventManager().subscribe(Events.PRINT_STARTED, self.onPrintStarted) - eventManager().subscribe(Events.PRINT_FAILED, self.onPrintDone) - eventManager().subscribe(Events.PRINT_DONE, self.onPrintDone) - eventManager().subscribe(Events.PRINT_RESUMED, self.onPrintResumed) - for (event, callback) in self.eventSubscriptions(): + eventManager().subscribe(Events.PRINT_STARTED, self.on_print_started) + eventManager().subscribe(Events.PRINT_FAILED, self.on_print_done) + eventManager().subscribe(Events.PRINT_DONE, self.on_print_done) + eventManager().subscribe(Events.PRINT_RESUMED, self.on_print_resumed) + for (event, callback) in self.event_subscriptions(): eventManager().subscribe(event, callback) - def postRoll(self): - return self._postRoll + @property + def post_roll(self): + return self._post_roll + @property def fps(self): return self._fps def unload(self): - if self._inTimelapse: - self.stopTimelapse(doCreateMovie=False) + if self._in_timelapse: + self.stop_timelapse(doCreateMovie=False) # unsubscribe events - eventManager().unsubscribe(Events.PRINT_STARTED, self.onPrintStarted) - eventManager().unsubscribe(Events.PRINT_FAILED, self.onPrintDone) - eventManager().unsubscribe(Events.PRINT_DONE, self.onPrintDone) - eventManager().unsubscribe(Events.PRINT_RESUMED, self.onPrintResumed) - for (event, callback) in self.eventSubscriptions(): + eventManager().unsubscribe(Events.PRINT_STARTED, self.on_print_started) + eventManager().unsubscribe(Events.PRINT_FAILED, self.on_print_done) + eventManager().unsubscribe(Events.PRINT_DONE, self.on_print_done) + eventManager().unsubscribe(Events.PRINT_RESUMED, self.on_print_resumed) + for (event, callback) in self.event_subscriptions(): eventManager().unsubscribe(event, callback) - def onPrintStarted(self, event, payload): + def on_print_started(self, event, payload): """ Override this to perform additional actions upon start of a print job. """ - self.startTimelapse(payload["file"]) + self.start_timelapse(payload["file"]) - def onPrintDone(self, event, payload): + def on_print_done(self, event, payload): """ Override this to perform additional actions upon the stop of a print job. """ - self.stopTimelapse(success=(event==Events.PRINT_DONE)) + self.stop_timelapse(success=(event==Events.PRINT_DONE)) - def onPrintResumed(self, event, payload): + def on_print_resumed(self, event, payload): """ Override this to perform additional actions upon the pausing of a print job. """ - if not self._inTimelapse: - self.startTimelapse(payload["file"]) + if not self._in_timelapse: + self.start_timelapse(payload["file"]) - def eventSubscriptions(self): + def event_subscriptions(self): """ Override this method to subscribe to additional events by returning an array of (event, callback) tuples. @@ -178,7 +189,7 @@ def eventSubscriptions(self): """ return [] - def configData(self): + def config_data(self): """ Override this method to return the current timelapse configuration data. The data should have the following form: @@ -188,94 +199,142 @@ def configData(self): """ return None - def startTimelapse(self, gcodeFile): + def start_timelapse(self, gcodeFile): self._logger.debug("Starting timelapse for %s" % gcodeFile) - self.cleanCaptureDir() + self.clean_capture_dir() - self._imageNumber = 0 - self._inTimelapse = True - self._gcodeFile = os.path.basename(gcodeFile) + self._image_number = 0 + self._in_timelapse = True + self._gcode_file = os.path.basename(gcodeFile) - def stopTimelapse(self, doCreateMovie=True, success=True): + def stop_timelapse(self, doCreateMovie=True, success=True): self._logger.debug("Stopping timelapse") - self._inTimelapse = False + self._in_timelapse = False def resetImageNumber(): - self._imageNumber = None + self._image_number = None def createMovie(): - self._renderThread = threading.Thread(target=self._createMovie, kwargs={"success": success}) - self._renderThread.daemon = True - self._renderThread.start() + self._render_thread = threading.Thread(target=self._create_movie, kwargs={"success": success}) + self._render_thread.daemon = True + self._render_thread.start() def resetAndCreate(): resetImageNumber() createMovie() - if self._postRoll > 0: - self._postRollStart = time.time() + def waitForCaptures(callback): + self._capture_queue.put(dict(type=self.__class__.QUEUE_ENTRY_TYPE_CALLBACK, callback=callback)) + + def getWaitForCaptures(callback): + def f(): + waitForCaptures(callback) + return f + + if self._post_roll > 0: + eventManager().fire(Events.POSTROLL_START, dict(postroll_duration=self.calculate_post_roll(), postroll_length=self.post_roll, postroll_fps=self.fps)) + self._post_roll_start = time.time() if doCreateMovie: - self._onPostRollDone = resetAndCreate + self._on_post_roll_done = getWaitForCaptures(resetAndCreate) else: - self._onPostRollDone = resetImageNumber - self.processPostRoll() + self._on_post_roll_done = resetImageNumber + self.process_post_roll() else: - self._postRollStart = None + self._post_roll_start = None if doCreateMovie: - resetAndCreate() + waitForCaptures(resetAndCreate) else: resetImageNumber() - def processPostRoll(self): - pass + def calculate_post_roll(self): + return None + + def process_post_roll(self): + self.post_roll_finished() + + def post_roll_finished(self): + if self.post_roll: + eventManager().fire(Events.POSTROLL_END) + if self._on_post_roll_done is not None: + self._on_post_roll_done() def captureImage(self): - if self._captureDir is None: + if self._capture_dir is None: self._logger.warn("Cannot capture image, capture directory is unset") return - if self._imageNumber is None: - self._logger.warn("Cannot capture image, image number is unset") - return + with self._capture_mutex: + if self._image_number is None: + self._logger.warn("Cannot capture image, image number is unset") + return + + filename = os.path.join(self._capture_dir, "tmp_%05d.jpg" % self._image_number) + self._image_number += 1 - with self._captureMutex: - filename = os.path.join(self._captureDir, "tmp_%05d.jpg" % self._imageNumber) - self._imageNumber += 1 self._logger.debug("Capturing image to %s" % filename) - captureThread = threading.Thread(target=self._captureWorker, kwargs={"filename": filename}) - captureThread.daemon = True - captureThread.start() + entry = dict(type=self.__class__.QUEUE_ENTRY_TYPE_CAPTURE, + filename=filename, + onerror=self._on_capture_error) + self._capture_queue.put(entry) return filename - def _captureWorker(self, filename): + def _on_capture_error(self): + with self._capture_mutex: + if self._image_number is not None and self._image_number > 0: + self._image_number -= 1 + + def _capture_queue_worker(self): + while self._capture_queue_active: + entry = self._capture_queue.get(block=True) + + if entry["type"] == self.__class__.QUEUE_ENTRY_TYPE_CAPTURE and "filename" in entry: + filename = entry["filename"] + onerror = entry.pop("onerror", None) + self._perform_capture(filename, onerror=onerror) + + elif entry["type"] == self.__class__.QUEUE_ENTRY_TYPE_CALLBACK and "callback" in entry: + args = entry.pop("args", []) + kwargs = entry.pop("kwargs", dict()) + entry["callback"](*args, **kwargs) + + def _perform_capture(self, filename, onerror=None): eventManager().fire(Events.CAPTURE_START, {"file": filename}) try: - urllib.urlretrieve(self._snapshotUrl, filename) - self._logger.debug("Image %s captured from %s" % (filename, self._snapshotUrl)) + self._logger.debug("Going to capture %s from %s" % (filename, self._snapshot_url)) + r = requests.get(self._snapshot_url, stream=True) + with open (filename, "wb") as f: + for chunk in r.iter_content(chunk_size=1024): + if chunk: + f.write(chunk) + f.flush() + self._logger.debug("Image %s captured from %s" % (filename, self._snapshot_url)) except: - self._logger.exception("Could not capture image %s from %s, decreasing image counter again" % (filename, self._snapshotUrl)) - with self._captureMutex: - if self._imageNumber is not None and self._imageNumber > 0: - self._imageNumber -= 1 - eventManager().fire(Events.CAPTURE_DONE, {"file": filename}) + self._logger.exception("Could not capture image %s from %s" % (filename, self._snapshot_url)) + if callable(onerror): + onerror() + eventManager().fire(Events.CAPTURE_FAILED, {"file": filename}) + return False + else: + eventManager().fire(Events.CAPTURE_DONE, {"file": filename}) + return True - def _createMovie(self, success=True): + def _create_movie(self, success=True): ffmpeg = settings().get(["webcam", "ffmpeg"]) bitrate = settings().get(["webcam", "bitrate"]) if ffmpeg is None or bitrate is None: self._logger.warn("Cannot create movie, path to ffmpeg or desired bitrate is unset") return - input = os.path.join(self._captureDir, "tmp_%05d.jpg") + input = os.path.join(self._capture_dir, "tmp_%05d.jpg") if success: - output = os.path.join(self._movieDir, "%s_%s.mpg" % (os.path.splitext(self._gcodeFile)[0], time.strftime("%Y%m%d%H%M%S"))) + output = os.path.join(self._movie_dir, "%s_%s.mpg" % (os.path.splitext(self._gcode_file)[0], time.strftime("%Y%m%d%H%M%S"))) else: - output = os.path.join(self._movieDir, "%s_%s-failed.mpg" % (os.path.splitext(self._gcodeFile)[0], time.strftime("%Y%m%d%H%M%S"))) + output = os.path.join(self._movie_dir, "%s_%s-failed.mpg" % (os.path.splitext(self._gcode_file)[0], time.strftime("%Y%m%d%H%M%S"))) # prepare ffmpeg command command = [ - ffmpeg, '-framerate', str(self._fps), '-loglevel', 'error', '-i', input, '-vcodec', 'mpeg2video', '-threads', str(self._ffmpegThreads), '-pix_fmt', 'yuv420p', '-r', str(self._fps), '-y', '-b', bitrate, + ffmpeg, '-framerate', str(self._fps), '-loglevel', 'error', '-i', input, '-vcodec', 'mpeg2video', '-threads', str(self._ffmpeg_threads), '-pix_fmt', 'yuv420p', '-r', str(self._fps), '-y', '-b', bitrate, '-f', 'vob'] filters = [] @@ -315,7 +374,7 @@ def _createMovie(self, success=True): # finalize command with output file self._logger.debug("Rendering movie to %s" % output) command.append("\"" + output + "\"") - eventManager().fire(Events.MOVIE_RENDERING, {"gcode": self._gcodeFile, "movie": output, "movie_basename": os.path.basename(output)}) + eventManager().fire(Events.MOVIE_RENDERING, {"gcode": self._gcode_file, "movie": output, "movie_basename": os.path.basename(output)}) command_str = " ".join(command) self._logger.debug("Executing command: %s" % command_str) @@ -323,75 +382,74 @@ def _createMovie(self, success=True): try: p = sarge.run(command_str, stderr=sarge.Capture()) if p.returncode == 0: - eventManager().fire(Events.MOVIE_DONE, {"gcode": self._gcodeFile, "movie": output, "movie_basename": os.path.basename(output)}) + eventManager().fire(Events.MOVIE_DONE, {"gcode": self._gcode_file, "movie": output, "movie_basename": os.path.basename(output)}) else: returncode = p.returncode stderr_text = p.stderr.text self._logger.warn("Could not render movie, got return code %r: %s" % (returncode, stderr_text)) - eventManager().fire(Events.MOVIE_FAILED, {"gcode": self._gcodeFile, "movie": output, "movie_basename": os.path.basename(output), "returncode": returncode, "error": stderr_text}) + eventManager().fire(Events.MOVIE_FAILED, {"gcode": self._gcode_file, "movie": output, "movie_basename": os.path.basename(output), "returncode": returncode, "error": stderr_text}) except: self._logger.exception("Could not render movie due to unknown error") - eventManager().fire(Events.MOVIE_FAILED, {"gcode": self._gcodeFile, "movie": output, "movie_basename": os.path.basename(output), "returncode": 255, "error": "Unknown error"}) + eventManager().fire(Events.MOVIE_FAILED, {"gcode": self._gcode_file, "movie": output, "movie_basename": os.path.basename(output), "returncode": 255, "error": "Unknown error"}) - def cleanCaptureDir(self): - if not os.path.isdir(self._captureDir): + def clean_capture_dir(self): + if not os.path.isdir(self._capture_dir): self._logger.warn("Cannot clean capture directory, it is unset") return - for filename in os.listdir(self._captureDir): + for filename in os.listdir(self._capture_dir): if not fnmatch.fnmatch(filename, "*.jpg"): continue - os.remove(os.path.join(self._captureDir, filename)) + os.remove(os.path.join(self._capture_dir, filename)) class ZTimelapse(Timelapse): - def __init__(self, postRoll=0, fps=25): - Timelapse.__init__(self, postRoll=postRoll, fps=fps) + def __init__(self, post_roll=0, fps=25): + Timelapse.__init__(self, post_roll=post_roll, fps=fps) self._logger.debug("ZTimelapse initialized") - def eventSubscriptions(self): + def event_subscriptions(self): return [ - (Events.Z_CHANGE, self._onZChange) + (Events.Z_CHANGE, self._on_z_change) ] - def configData(self): + def config_data(self): return { "type": "zchange" } - def processPostRoll(self): - Timelapse.processPostRoll(self) + def process_post_roll(self): + with self._capture_mutex: + filename = os.path.join(self._capture_dir, "tmp_%05d.jpg" % self._image_number) + self._image_number += 1 - filename = os.path.join(self._captureDir, "tmp_%05d.jpg" % self._imageNumber) - self._imageNumber += 1 - with self._captureMutex: - self._captureWorker(filename) + if self._perform_capture(filename): + for _ in range(self._post_roll * self._fps): + newFile = os.path.join(self._capture_dir, "tmp_%05d.jpg" % self._image_number) + self._image_number += 1 + shutil.copyfile(filename, newFile) - for i in range(self._postRoll * self._fps): - newFile = os.path.join(self._captureDir, "tmp_%05d.jpg" % (self._imageNumber)) - self._imageNumber += 1 - shutil.copyfile(filename, newFile) + Timelapse.process_post_roll(self) - if self._onPostRollDone is not None: - self._onPostRollDone() - - def _onZChange(self, event, payload): + def _on_z_change(self, event, payload): self.captureImage() class TimedTimelapse(Timelapse): - def __init__(self, postRoll=0, interval=1, fps=25): - Timelapse.__init__(self, postRoll=postRoll, fps=fps) + def __init__(self, post_roll=0, interval=1, fps=25): + Timelapse.__init__(self, post_roll=post_roll, fps=fps) self._interval = interval if self._interval < 1: self._interval = 1 # force minimum interval of 1s - self._timerThread = None + self._postroll_captures = 0 + self._timer = None self._logger.debug("TimedTimelapse initialized") + @property def interval(self): return self._interval - def configData(self): + def config_data(self): return { "type": "timed", "options": { @@ -399,25 +457,39 @@ def configData(self): } } - def onPrintStarted(self, event, payload): - Timelapse.onPrintStarted(self, event, payload) - if self._timerThread is not None: + def on_print_started(self, event, payload): + Timelapse.on_print_started(self, event, payload) + if self._timer is not None: return - self._timerThread = threading.Thread(target=self._timerWorker) - self._timerThread.daemon = True - self._timerThread.start() + self._logger.debug("Starting timer for interval based timelapse") + from octoprint.util import RepeatedTimer + self._timer = RepeatedTimer(self._interval, self._timer_task, + run_first=True, condition=self._timer_active, + on_finish=self._on_timer_finished) + self._timer.start() - def onPrintDone(self, event, payload): - Timelapse.onPrintDone(self, event, payload) - self._timerThread = None + def on_print_done(self, event, payload): + self._postroll_captures = self.post_roll * self.fps + Timelapse.on_print_done(self, event, payload) - def _timerWorker(self): - self._logger.debug("Starting timer for interval based timelapse") - while self._inTimelapse or (self._postRollStart and time.time() - self._postRollStart <= self._postRoll * self._fps): - self.captureImage() - time.sleep(self._interval) + def calculate_post_roll(self): + return self.post_roll * self.fps * self.interval + + def process_post_roll(self): + pass + + def post_roll_finished(self): + Timelapse.post_roll_finished(self) + self._timer = None + + def _timer_active(self): + return self._in_timelapse or self._postroll_captures > 0 + + def _timer_task(self): + self.captureImage() + if self._postroll_captures > 0: + self._postroll_captures -= 1 - if self._postRollStart is not None and self._onPostRollDone is not None: - self._onPostRollDone() - self._postRollStart = None + def _on_timer_finished(self): + self.post_roll_finished() diff --git a/src/octoprint/users.py b/src/octoprint/users.py index fc5be2b75..d4890a642 100644 --- a/src/octoprint/users.py +++ b/src/octoprint/users.py @@ -17,29 +17,37 @@ from octoprint.settings import settings +from octoprint.util import atomic_write + class UserManager(object): valid_roles = ["user", "admin"] def __init__(self): self._logger = logging.getLogger(__name__) self._session_users_by_session = dict() - self._session_users_by_username = dict() + self._session_users_by_userid = dict() def login_user(self, user): self._cleanup_sessions() - if user is None \ - or (isinstance(user, LocalProxy) and not isinstance(user._get_current_object(), User)) \ - or (not isinstance(user, LocalProxy) and not isinstance(user, User)): + if user is None: + return + + if isinstance(user, LocalProxy): + user = user._get_current_object() + + if not isinstance(user, User): return None if not isinstance(user, SessionUser): user = SessionUser(user) + self._session_users_by_session[user.get_session()] = user - if not user.get_name() in self._session_users_by_username: - self._session_users_by_username[user.get_name()] = [] - self._session_users_by_username[user.get_name()].append(user) + userid = user.get_id() + if not userid in self._session_users_by_userid: + self._session_users_by_userid[userid] = [] + self._session_users_by_userid[userid].append(user) self._logger.debug("Logged in user: %r" % user) @@ -49,14 +57,18 @@ def logout_user(self, user): if user is None: return + if isinstance(user, LocalProxy): + user = user._get_current_object() + if not isinstance(user, SessionUser): return - if user.get_name() in self._session_users_by_username: - users_by_username = self._session_users_by_username[user.get_name()] - for u in users_by_username: + userid = user.get_id() + if userid in self._session_users_by_userid: + users_by_userid = self._session_users_by_userid[userid] + for u in users_by_userid: if u.get_session() == user.get_session(): - users_by_username.remove(u) + users_by_userid.remove(u) break if user.get_session() in self._session_users_by_session: @@ -137,21 +149,19 @@ def changeUserSettings(self, username, new_settings): pass def removeUser(self, username): - if username in self._session_users_by_username: - users = self._session_users_by_username[username] + if username in self._session_users_by_userid: + users = self._session_users_by_userid[username] sessions = [user.get_session() for user in users if isinstance(user, SessionUser)] for session in sessions: if session in self._session_users_by_session: del self._session_users_by_session[session] - del self._session_users_by_username[username] + del self._session_users_by_userid[username] - def findUser(self, username=None, session=None): - if session is not None: - for session in self._session_users_by_session: - user = self._session_users_by_session[session] - if username is None or username == user.get_name(): - return user - break + def findUser(self, userid=None, session=None): + if session is not None and session in self._session_users_by_session: + user = self._session_users_by_session[session] + if userid is None or userid == user.get_id(): + return user return None @@ -209,7 +219,7 @@ def _save(self, force=False): "settings": user._settings } - with open(self._userfile, "wb") as f: + with atomic_write(self._userfile, "wb") as f: yaml.safe_dump(data, f, default_flow_style=False, indent=" ", allow_unicode=True) self._dirty = False self._load() @@ -345,16 +355,16 @@ def removeUser(self, username): self._dirty = True self._save() - def findUser(self, username=None, apikey=None, session=None): - user = UserManager.findUser(self, username=username, session=session) + def findUser(self, userid=None, apikey=None, session=None): + user = UserManager.findUser(self, userid=userid, session=session) if user is not None: return user - if username is not None: - if username not in self._users.keys(): + if userid is not None: + if userid not in self._users.keys(): return None - return self._users[username] + return self._users[userid] elif apikey is not None: for user in self._users.values(): @@ -413,7 +423,7 @@ def check_password(self, passwordHash): return self._passwordHash == passwordHash def get_id(self): - return self._username + return self.get_name() def get_name(self): return self._username diff --git a/src/octoprint/util/__init__.py b/src/octoprint/util/__init__.py index 07f80b813..bc4ddfd71 100644 --- a/src/octoprint/util/__init__.py +++ b/src/octoprint/util/__init__.py @@ -505,6 +505,29 @@ def atomic_write(filename, mode="w+b", prefix="tmp", suffix=""): shutil.move(temp_config.name, filename) +def bom_aware_open(filename, encoding="ascii", mode="r", **kwargs): + import codecs + + codec = codecs.lookup(encoding) + encoding = codec.name + + if kwargs is None: + kwargs = dict() + + potential_bom_attribute = "BOM_" + codec.name.replace("utf-", "utf").upper() + if "r" in mode and hasattr(codecs, potential_bom_attribute): + # these encodings might have a BOM, so let's see if there is one + bom = getattr(codecs, potential_bom_attribute) + + with open(filename, "rb") as f: + header = f.read(4) + + if header.startswith(bom): + encoding += "-sig" + + return codecs.open(filename, encoding=encoding, **kwargs) + + class RepeatedTimer(threading.Thread): """ This class represents an action that should be run repeatedly in an interval. It is similar to python's @@ -562,10 +585,18 @@ def hello(): run_first (boolean): If set to True, the function will be run for the first time *before* the first wait period. If set to False (the default), the function will be run for the first time *after* the first wait period. condition (callable): Condition that needs to be True for loop to continue. Defaults to ``lambda: True``. + on_condition_false (callable): Callback to call when the timer finishes due to condition becoming false. Will + be called before the ``on_finish`` callback. + on_cancelled (callable): Callback to call when the timer finishes due to being cancelled. Will be called + before the ``on_finish`` callback. + on_finish (callable): Callback to call when the timer finishes, either due to being cancelled or since + the condition became false. daemon (bool): daemon flag to set on underlying thread. """ - def __init__(self, interval, function, args=None, kwargs=None, run_first=False, condition=None, daemon=True): + def __init__(self, interval, function, args=None, kwargs=None, + run_first=False, condition=None, on_condition_false=None, + on_cancelled=None, on_finish=None, daemon=True): threading.Thread.__init__(self) if args is None: @@ -586,10 +617,15 @@ def __init__(self, interval, function, args=None, kwargs=None, run_first=False, self.kwargs = kwargs self.run_first = run_first self.condition = condition + + self.on_condition_false = on_condition_false + self.on_cancelled = on_cancelled + self.on_finish = on_finish + self.daemon = daemon def cancel(self): - self.finished.set() + self._finish(self.on_cancelled) def run(self): while self.condition(): @@ -604,15 +640,26 @@ def run(self): # wait, but break if we are cancelled self.finished.wait(self.interval()) if self.finished.is_set(): - break + return if not self.run_first: # if we are to run the function AFTER waiting for the first time self.function(*self.args, **self.kwargs) - # make sure we set our finished event so we can detect that the loop was finished + # we'll only get here if the condition was false + self._finish(self.on_condition_false) + + def _finish(self, *callbacks): self.finished.set() + for callback in callbacks: + if not callable(callback): + continue + callback() + + if callable(self.on_finish): + self.on_finish() + class CountedEvent(object): diff --git a/src/octoprint/util/comm.py b/src/octoprint/util/comm.py index 49c8bc6c0..9bc2c7024 100644 --- a/src/octoprint/util/comm.py +++ b/src/octoprint/util/comm.py @@ -24,7 +24,7 @@ from octoprint.events import eventManager, Events from octoprint.filemanager import valid_file_type from octoprint.filemanager.destinations import FileDestinations -from octoprint.util import get_exception_string, sanitize_ascii, filter_non_ascii, CountedEvent, RepeatedTimer +from octoprint.util import get_exception_string, sanitize_ascii, filter_non_ascii, CountedEvent, RepeatedTimer, to_unicode, bom_aware_open try: import _winreg @@ -491,7 +491,7 @@ def fakeOk(self): self._clear_to_send.set() def sendCommand(self, cmd, cmd_type=None, processed=False): - cmd = cmd.encode('ascii', 'replace') + cmd = to_unicode(cmd, errors="replace") if not processed: cmd = process_gcode_line(cmd) if not cmd: @@ -1549,10 +1549,11 @@ def _send_loop(self): continue # now comes the part where we increase line numbers and send stuff - no turning back now + command_to_send = command.encode("ascii", errors="replace") if (gcode is not None or self._sendChecksumWithUnknownCommands) and (self.isPrinting() or self._alwaysSendChecksum): - self._doIncrementAndSendWithChecksum(command) + self._doIncrementAndSendWithChecksum(command_to_send) else: - self._doSendWithoutChecksum(command) + self._doSendWithoutChecksum(command_to_send) # trigger "sent" phase and use up one "ok" self._process_command_phase("sent", command, command_type, gcode=gcode) @@ -1952,7 +1953,7 @@ def start(self): Opens the file for reading and determines the file size. """ PrintingFileInformation.start(self) - self._handle = open(self._filename, "r") + self._handle = bom_aware_open(self._filename, encoding="utf-8", errors="replace") def close(self): """ @@ -1982,7 +1983,7 @@ def getNext(self): if self._handle is None: # file got closed just now return None - line = self._handle.readline() + line = to_unicode(self._handle.readline()) if not line: self.close() processed = process_gcode_line(line, offsets=offsets, current_tool=current_tool) diff --git a/src/octoprint/util/gcodeInterpreter.py b/src/octoprint/util/gcodeInterpreter.py index b0b69b665..853cbfd62 100644 --- a/src/octoprint/util/gcodeInterpreter.py +++ b/src/octoprint/util/gcodeInterpreter.py @@ -35,7 +35,9 @@ def load(self, filename, printer_profile, throttle=None): if os.path.isfile(filename): self.filename = filename self._fileSize = os.stat(filename).st_size - with open(filename, "r") as f: + + import codecs + with codecs.open(filename, encoding="utf-8", errors="replace") as f: self._load(f, printer_profile, throttle=throttle) def abort(self): diff --git a/src/octoprint/util/pip.py b/src/octoprint/util/pip.py index 980e5735c..60ceb1b68 100644 --- a/src/octoprint/util/pip.py +++ b/src/octoprint/util/pip.py @@ -21,14 +21,16 @@ class PipCaller(object): def __init__(self, configured=None): self._logger = logging.getLogger(__name__) - self._configured = configured + self.configured = configured + self.refresh = False self._command = None self._version = None + self._version_string = None + self._use_sudo = False - self._command, self._version = self._find_pip() + self.trigger_refresh() - self.refresh = False self.on_log_call = lambda *args, **kwargs: None self.on_log_stdout = lambda *args, **kwargs: None self.on_log_stderr = lambda *args, **kwargs: None @@ -53,19 +55,37 @@ def command(self): def version(self): return self._version + @property + def version_string(self): + return self._version_string + + @property + def use_sudo(self): + return self._use_sudo + @property def available(self): return self._command is not None + def trigger_refresh(self): + try: + self._command, self._version, self._version_string, self._use_sudo = self._find_pip() + except: + self._logger.exception("Error while discovering pip command") + self._command = None + self._version = None + self.refresh = False + def execute(self, *args): if self.refresh: - self._command, self._version = self._find_pip() - self.refresh = False + self.trigger_refresh() if self._command is None: raise UnknownPip() command = [self._command] + list(args) + if self._use_sudo: + command = ["sudo"] + command joined_command = " ".join(command) self._logger.debug(u"Calling: {}".format(joined_command)) @@ -111,8 +131,16 @@ def execute(self, *args): def _find_pip(self): - pip_command = self._configured + pip_command = self.configured + + if pip_command is not None and pip_command.startswith("sudo "): + pip_command = pip_command[len("sudo "):] + pip_sudo = True + else: + pip_sudo = False + pip_version = None + version_segment = None if pip_command is None: import os @@ -144,7 +172,11 @@ def _find_pip(self): if pip_command is not None: self._logger.debug("Found pip at {}, going to figure out its version".format(pip_command)) - p = sarge.run([pip_command, "--version"], stdout=sarge.Capture(), stderr=sarge.Capture()) + + sarge_command = [pip_command, "--version"] + if pip_sudo: + sarge_command = ["sudo"] + sarge_command + p = sarge.run(sarge_command, stdout=sarge.Capture(), stderr=sarge.Capture()) if p.returncode != 0: self._logger.warn("Error while trying to run pip --version: {}".format(p.stderr.text)) @@ -174,7 +206,7 @@ def _find_pip(self): else: self._logger.info("Found pip at {}, version is {}".format(pip_command, version_segment)) - return pip_command, pip_version + return pip_command, pip_version, version_segment, pip_sudo def _log_stdout(self, *lines): self.on_log_stdout(*lines) diff --git a/tests/filemanager/__init__.py b/tests/filemanager/__init__.py index 65f330577..92a8db729 100644 --- a/tests/filemanager/__init__.py +++ b/tests/filemanager/__init__.py @@ -8,4 +8,3 @@ __author__ = "Gina Häußge " __license__ = 'GNU Affero General Public License http://www.gnu.org/licenses/agpl.html' __copyright__ = "Copyright (C) 2014 The OctoPrint Project - Released under terms of the AGPLv3 License" - diff --git a/tests/filemanager/test_filemanager.py b/tests/filemanager/test_filemanager.py index 110df373f..68d49ecd3 100644 --- a/tests/filemanager/test_filemanager.py +++ b/tests/filemanager/test_filemanager.py @@ -13,6 +13,98 @@ import octoprint.filemanager import octoprint.filemanager.util +class FilemanagerMethodTest(unittest.TestCase): + + def setUp(self): + # mock plugin manager + self.plugin_manager_patcher = mock.patch("octoprint.plugin.plugin_manager") + self.plugin_manager_getter = self.plugin_manager_patcher.start() + + self.plugin_manager = mock.MagicMock() + + hook_extensions = dict( + some_plugin=lambda: dict(dict(machinecode=dict(foo=["foo", "f"]))), + other_plugin=lambda: dict(dict(model=dict(amf=["amf"]))), + mime_map=lambda: dict( + mime_map=dict( + mime_map_yes=octoprint.filemanager.ContentTypeMapping(["mime_map_yes"], "application/mime_map_yes") + ) + ), + mime_detect=lambda: dict( + dict( + machinecode=dict( + mime_detect_yes=octoprint.filemanager.ContentTypeDetector(["mime_detect_yes"], lambda x: "application/mime_detect_yes"), + mime_detect_no=octoprint.filemanager.ContentTypeDetector(["mime_detect_no"], lambda x: None) + ) + ) + ) + ) + self.plugin_manager.get_hooks.return_value = hook_extensions + + self.plugin_manager_getter.return_value = self.plugin_manager + + def tearDown(self): + self.plugin_manager_patcher.stop() + + def test_full_extension_tree(self): + full = octoprint.filemanager.full_extension_tree() + self.assertTrue("machinecode" in full) + self.assertTrue("gcode" in full["machinecode"]) + self.assertTrue(isinstance(full["machinecode"]["gcode"], octoprint.filemanager.ContentTypeMapping)) + self.assertItemsEqual(["gcode", "gco", "g"], full["machinecode"]["gcode"].extensions) + self.assertTrue("foo" in full["machinecode"]) + self.assertTrue(isinstance(full["machinecode"]["foo"], list)) + self.assertItemsEqual(["f", "foo"], full["machinecode"]["foo"]) + + self.assertTrue("model" in full) + self.assertTrue("stl" in full["model"]) + self.assertTrue(isinstance(full["model"]["stl"], octoprint.filemanager.ContentTypeMapping)) + self.assertItemsEqual(["stl"], full["model"]["stl"].extensions) + self.assertTrue("amf" in full["model"]) + self.assertTrue(isinstance(full["model"]["amf"], list)) + self.assertItemsEqual(["amf"], full["model"]["amf"]) + + def test_get_mimetype(self): + self.assertEquals(octoprint.filemanager.get_mime_type("foo.stl"), "application/sla") + self.assertEquals(octoprint.filemanager.get_mime_type("foo.gcode"), "text/plain") + self.assertEquals(octoprint.filemanager.get_mime_type("foo.unknown"), "application/octet-stream") + self.assertEquals(octoprint.filemanager.get_mime_type("foo.mime_map_yes"), "application/mime_map_yes") + self.assertEquals(octoprint.filemanager.get_mime_type("foo.mime_map_no"), "application/octet-stream") + self.assertEquals(octoprint.filemanager.get_mime_type("foo.mime_detect_yes"), "application/mime_detect_yes") + self.assertEquals(octoprint.filemanager.get_mime_type("foo.mime_detect_no"), "application/octet-stream") + + def test_valid_file_type(self): + self.assertTrue(octoprint.filemanager.valid_file_type("foo.stl", type="model")) + self.assertTrue(octoprint.filemanager.valid_file_type("foo.stl", type="stl")) + self.assertFalse(octoprint.filemanager.valid_file_type("foo.stl", type="machinecode")) + self.assertTrue(octoprint.filemanager.valid_file_type("foo.foo", type="machinecode")) + self.assertTrue(octoprint.filemanager.valid_file_type("foo.foo", type="foo")) + self.assertTrue(octoprint.filemanager.valid_file_type("foo.foo")) + self.assertTrue(octoprint.filemanager.valid_file_type("foo.mime_map_yes")) + self.assertTrue(octoprint.filemanager.valid_file_type("foo.mime_detect_yes")) + self.assertFalse(octoprint.filemanager.valid_file_type("foo.unknown")) + + def test_get_file_type(self): + self.assertEquals(["machinecode", "gcode"], octoprint.filemanager.get_file_type("foo.gcode")) + self.assertEquals(["machinecode", "gcode"], octoprint.filemanager.get_file_type("foo.gco")) + self.assertEquals(["machinecode", "foo"], octoprint.filemanager.get_file_type("foo.f")) + self.assertEquals(["model", "stl"], octoprint.filemanager.get_file_type("foo.stl")) + self.assertEquals(["model", "amf"], octoprint.filemanager.get_file_type("foo.amf")) + self.assertIsNone(octoprint.filemanager.get_file_type("foo.unknown")) + + def test_hook_failure(self): + def hook(): + raise RuntimeError("Boo!") + self.plugin_manager.get_hooks.return_value = dict(hook=hook) + + with mock.patch("octoprint.filemanager.logging") as patched_logging: + logger = mock.MagicMock() + patched_logging.getLogger.return_value = logger + + octoprint.filemanager.get_all_extensions() + + self.assertEquals(1, len(logger.mock_calls)) + class FileManagerTest(unittest.TestCase): def setUp(self): @@ -109,13 +201,13 @@ def test_get_metadata(self): self.assertEquals(metadata, expected) self.local_storage.get_metadata.assert_called_once_with("test.file") - @mock.patch("__builtin__.open", new_callable=mock.mock_open) + @mock.patch("octoprint.filemanager.util.atomic_write") @mock.patch("io.FileIO") @mock.patch("shutil.copyfileobj") @mock.patch("os.remove") @mock.patch("tempfile.NamedTemporaryFile") @mock.patch("time.time", side_effect=[1411979916.422, 1411979932.116]) - def test_slice(self, mocked_time, mocked_tempfile, mocked_os, mocked_shutil, mocked_fileio, mocked_open): + def test_slice(self, mocked_time, mocked_tempfile, mocked_os, mocked_shutil, mocked_fileio, mocked_atomic_write): callback = mock.MagicMock() callback_args = ("one", "two", "three") @@ -187,8 +279,8 @@ def slice(slicer_name, source_path, dest_path, profile, done_cb, printer_profile self.local_storage.add_file.assert_called_once_with("dest.file", mock.ANY, printer_profile=expected_printer_profile, allow_overwrite=True, links=expected_links) # assert that the generated gcode was manipulated as required - expected_open_calls = [mock.call("prefix/dest.file", "wb")] - self.assertEquals(mocked_open.call_args_list, expected_open_calls) + expected_atomic_write_calls = [mock.call("prefix/dest.file", "wb")] + self.assertEquals(mocked_atomic_write.call_args_list, expected_atomic_write_calls) #mocked_open.return_value.write.assert_called_once_with(";Generated from source.file aabbccddeeff\r") # assert that shutil was asked to copy the concatenated multistream diff --git a/tests/util/test_repeated_timer.py b/tests/util/test_repeated_timer.py new file mode 100644 index 000000000..6eed773b9 --- /dev/null +++ b/tests/util/test_repeated_timer.py @@ -0,0 +1,180 @@ +# coding=utf-8 +from __future__ import absolute_import + +__author__ = "Gina Häußge " +__license__ = 'GNU Affero General Public License http://www.gnu.org/licenses/agpl.html' +__copyright__ = "Copyright (C) 2015 The OctoPrint Project - Released under terms of the AGPLv3 License" + +import unittest +import mock +import time + +from octoprint.util import RepeatedTimer + + +class Countdown(object): + def __init__(self, start): + self._counter = start + + def step(self): + self._counter -= 1 + + @property + def counter(self): + return self._counter + + +class IncreasingInterval(Countdown): + def __init__(self, start, factor): + Countdown.__init__(self, start) + self._start = start + self._factor = factor + + def interval(self): + result = (self._start - self._counter + 1) * self._factor + return result + + +class RepeatedTimerTest(unittest.TestCase): + + def setUp(self): + pass + + def test_condition(self): + countdown = Countdown(5) + timer_task = mock.MagicMock() + timer_task.side_effect = countdown.step + + timer = RepeatedTimer(0.1, timer_task, condition=lambda: countdown.counter > 0) + timer.start() + + # wait for it + timer.join() + + self.assertEquals(5, timer_task.call_count) + + def test_finished_callback(self): + countdown = Countdown(5) + timer_task = mock.MagicMock() + timer_task.side_effect = countdown.step + + on_finished = mock.MagicMock() + + timer = RepeatedTimer(0.1, timer_task, condition=lambda: countdown.counter > 0, on_finish=on_finished) + timer.start() + + # wait for it + timer.join() + + self.assertEquals(1, on_finished.call_count) + + def test_condition_callback(self): + countdown = Countdown(5) + timer_task = mock.MagicMock() + timer_task.side_effect = countdown.step + + on_cancelled = mock.MagicMock() + on_condition_false = mock.MagicMock() + + timer = RepeatedTimer(0.1, timer_task, + condition=lambda: countdown.counter > 0, + on_condition_false=on_condition_false, + on_cancelled=on_cancelled) + timer.start() + + # wait for it + timer.join() + + self.assertEquals(1, on_condition_false.call_count) + self.assertEquals(0, on_cancelled.call_count) + + def test_cancelled_callback(self): + countdown = Countdown(5) + timer_task = mock.MagicMock() + timer_task.side_effect = countdown.step + + on_cancelled = mock.MagicMock() + on_condition_false = mock.MagicMock() + + timer = RepeatedTimer(10, timer_task, + condition=lambda: countdown.counter > 0, + on_condition_false=on_condition_false, + on_cancelled=on_cancelled) + timer.start() + + # give it some time to run + time.sleep(1) + + # then cancel it and wait for the thread to really finish + timer.cancel() + timer.join() + + self.assertEquals(0, on_condition_false.call_count) + self.assertEquals(1, on_cancelled.call_count) + + def test_run_first(self): + timer_task = mock.MagicMock() + + timer = RepeatedTimer(60, timer_task, run_first=True) + timer.start() + + # give it some time to run + time.sleep(1) + + # then cancel it and wait for the thread to really finish + timer.cancel() + timer.join() + + # should have run once + self.assertEquals(1, timer_task.call_count) + + def test_not_run_first(self): + timer_task = mock.MagicMock() + + timer = RepeatedTimer(60, timer_task) + timer.start() + + # give it some time to run - should hang in the sleep phase though + time.sleep(1) + + # then cancel it and wait for the thread to really finish + timer.cancel() + timer.join() + + self.assertEquals(0, timer_task.call_count) + + def test_adjusted_interval(self): + increasing_interval = IncreasingInterval(3, 1) + + timer_task = mock.MagicMock() + timer_task.side_effect = increasing_interval.step + + timer = RepeatedTimer(increasing_interval.interval, + timer_task, + condition=lambda: increasing_interval.counter > 0) + + # this should take 1 + 2 + 3 = 6s + start_time = time.time() + timer.start() + timer.join() + duration = time.time() - start_time + + self.assertEquals(3, timer_task.call_count) + self.assertGreaterEqual(duration, 6) + self.assertLess(duration, 7) + + def test_condition_change_during_task(self): + def sleep(): + time.sleep(2) + + timer_task = mock.MagicMock() + timer_task.side_effect = sleep + + timer = RepeatedTimer(0.1, timer_task, run_first=True) + timer.start() + + time.sleep(1) + timer.condition = lambda: False + timer.join() + + self.assertEquals(1, timer_task.call_count)