From a2e5fc0c5c38cbdf5b089e0ce9e8f9df0bb6d39f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Wed, 2 Sep 2015 20:18:18 +0200 Subject: [PATCH 01/34] maintenance branch is now 1.2.7-dev --- .versioneer-lookup | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.versioneer-lookup b/.versioneer-lookup index 9174f0ca6..41fd3a301 100644 --- a/.versioneer-lookup +++ b/.versioneer-lookup @@ -10,10 +10,10 @@ # master shall not use the lookup table, only tags master -# maintenance is currently the branch for preparation of maintenance release 1.2.6 +# maintenance is currently the branch for preparation of maintenance release 1.2.7 # so are any fix/... branches -maintenance 1.2.6-dev 96fc70bdb2dd74ba04c3071f70da385b0408904a -fix/.* 1.2.6-dev 96fc70bdb2dd74ba04c3071f70da385b0408904a +maintenance 1.2.7-dev 536bb31965db17b969e7c1c53e241ddac4ae1814 +fix/.* 1.2.7-dev 536bb31965db17b969e7c1c53e241ddac4ae1814 # Special case disconnected checkouts, e.g. 'git checkout ' \(detached.* From cd7ac032f454f3182e9bd1aa7b45dc041e0e13fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Wed, 9 Sep 2015 16:13:10 +0200 Subject: [PATCH 02/34] Fixed an issue that cause user sessions to not be properly associated Sessions could get duplicated, wrongly saved etc. The reason was not persisting the actual user object to the internal session map (but the LocalProxy instead). That could lead to multiple sessions being created for one login, or the session user being set to an anonymous user, or various other odd effects depending on timing. (cherry picked from commit 8aeac51) --- src/octoprint/server/util/flask.py | 6 ++++-- src/octoprint/users.py | 24 +++++++++++++++--------- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/src/octoprint/server/util/flask.py b/src/octoprint/server/util/flask.py index 30136222e..b67a09080 100644 --- a/src/octoprint/server/util/flask.py +++ b/src/octoprint/server/util/flask.py @@ -227,8 +227,10 @@ def passive_login(): user = flask.ext.login.current_user if user is not None and not user.is_anonymous(): - flask.g.user = user flask.ext.principal.identity_changed.send(flask.current_app._get_current_object(), identity=flask.ext.principal.Identity(user.get_id())) + if hasattr(user, "get_session"): + flask.session["usersession.id"] = user.get_session() + flask.g.user = user return flask.jsonify(user.asDict()) elif settings().getBoolean(["accessControl", "autologinLocal"]) \ and settings().get(["accessControl", "autologinAs"]) is not None \ @@ -252,7 +254,7 @@ def passive_login(): logger = logging.getLogger(__name__) logger.exception("Could not autologin user %s for networks %r" % (autologinAs, localNetworks)) - return ("", 204) + return "", 204 #~~ cache decorator for cacheable views diff --git a/src/octoprint/users.py b/src/octoprint/users.py index fc5be2b75..7c40d2d8c 100644 --- a/src/octoprint/users.py +++ b/src/octoprint/users.py @@ -28,13 +28,18 @@ def __init__(self): def login_user(self, user): self._cleanup_sessions() - if user is None \ - or (isinstance(user, LocalProxy) and not isinstance(user._get_current_object(), User)) \ - or (not isinstance(user, LocalProxy) and not isinstance(user, User)): + if user is None: + return + + if isinstance(user, LocalProxy): + user = user._get_current_object() + + if not isinstance(user, User): return None if not isinstance(user, SessionUser): user = SessionUser(user) + self._session_users_by_session[user.get_session()] = user if not user.get_name() in self._session_users_by_username: @@ -49,6 +54,9 @@ def logout_user(self, user): if user is None: return + if isinstance(user, LocalProxy): + user = user._get_current_object() + if not isinstance(user, SessionUser): return @@ -146,12 +154,10 @@ def removeUser(self, username): del self._session_users_by_username[username] def findUser(self, username=None, session=None): - if session is not None: - for session in self._session_users_by_session: - user = self._session_users_by_session[session] - if username is None or username == user.get_name(): - return user - break + if session is not None and session in self._session_users_by_session: + user = self._session_users_by_session[session] + if username is None or username == user.get_id(): + return user return None From 5c9b507cb7985f2adae3c822b390a7ef4982efbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Fri, 11 Sep 2015 08:34:05 +0200 Subject: [PATCH 03/34] User user id, not user name, for all user operations (cherry picked from commit 7021b9f) --- src/octoprint/server/__init__.py | 6 ++--- src/octoprint/users.py | 40 +++++++++++++++++--------------- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/src/octoprint/server/__init__.py b/src/octoprint/server/__init__.py index 18ea638df..646e573d7 100644 --- a/src/octoprint/server/__init__.py +++ b/src/octoprint/server/__init__.py @@ -81,7 +81,7 @@ def on_identity_loaded(sender, identity): if user is None: return - identity.provides.add(UserNeed(user.get_name())) + identity.provides.add(UserNeed(user.get_id())) if user.is_user(): identity.provides.add(RoleNeed("user")) if user.is_admin(): @@ -98,9 +98,9 @@ def load_user(id): if userManager is not None: if sessionid: - return userManager.findUser(username=id, session=sessionid) + return userManager.findUser(userid=id, session=sessionid) else: - return userManager.findUser(username=id) + return userManager.findUser(userid=id) return users.DummyUser() diff --git a/src/octoprint/users.py b/src/octoprint/users.py index 7c40d2d8c..76bf23bd2 100644 --- a/src/octoprint/users.py +++ b/src/octoprint/users.py @@ -23,7 +23,7 @@ class UserManager(object): def __init__(self): self._logger = logging.getLogger(__name__) self._session_users_by_session = dict() - self._session_users_by_username = dict() + self._session_users_by_userid = dict() def login_user(self, user): self._cleanup_sessions() @@ -42,9 +42,10 @@ def login_user(self, user): self._session_users_by_session[user.get_session()] = user - if not user.get_name() in self._session_users_by_username: - self._session_users_by_username[user.get_name()] = [] - self._session_users_by_username[user.get_name()].append(user) + userid = user.get_id() + if not userid in self._session_users_by_userid: + self._session_users_by_userid[userid] = [] + self._session_users_by_userid[userid].append(user) self._logger.debug("Logged in user: %r" % user) @@ -60,11 +61,12 @@ def logout_user(self, user): if not isinstance(user, SessionUser): return - if user.get_name() in self._session_users_by_username: - users_by_username = self._session_users_by_username[user.get_name()] - for u in users_by_username: + userid = user.get_id() + if userid in self._session_users_by_userid: + users_by_userid = self._session_users_by_userid[userid] + for u in users_by_userid: if u.get_session() == user.get_session(): - users_by_username.remove(u) + users_by_userid.remove(u) break if user.get_session() in self._session_users_by_session: @@ -145,18 +147,18 @@ def changeUserSettings(self, username, new_settings): pass def removeUser(self, username): - if username in self._session_users_by_username: - users = self._session_users_by_username[username] + if username in self._session_users_by_userid: + users = self._session_users_by_userid[username] sessions = [user.get_session() for user in users if isinstance(user, SessionUser)] for session in sessions: if session in self._session_users_by_session: del self._session_users_by_session[session] - del self._session_users_by_username[username] + del self._session_users_by_userid[username] - def findUser(self, username=None, session=None): + def findUser(self, userid=None, session=None): if session is not None and session in self._session_users_by_session: user = self._session_users_by_session[session] - if username is None or username == user.get_id(): + if userid is None or userid == user.get_id(): return user return None @@ -351,16 +353,16 @@ def removeUser(self, username): self._dirty = True self._save() - def findUser(self, username=None, apikey=None, session=None): - user = UserManager.findUser(self, username=username, session=session) + def findUser(self, userid=None, apikey=None, session=None): + user = UserManager.findUser(self, userid=userid, session=session) if user is not None: return user - if username is not None: - if username not in self._users.keys(): + if userid is not None: + if userid not in self._users.keys(): return None - return self._users[username] + return self._users[userid] elif apikey is not None: for user in self._users.values(): @@ -419,7 +421,7 @@ def check_password(self, passwordHash): return self._passwordHash == passwordHash def get_id(self): - return self._username + return self.get_name() def get_name(self): return self._username From b56ba6589c9710543d2796c1ce95acd6152b886d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Fri, 11 Sep 2015 08:14:35 +0200 Subject: [PATCH 04/34] Ignore update definitions that are lacking the type Caused a KeyError so far, update definitions that are broken like that will now just be ignored instead. Closes #1057 (cherry picked from commit 2efc5c4) --- src/octoprint/plugins/softwareupdate/__init__.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/octoprint/plugins/softwareupdate/__init__.py b/src/octoprint/plugins/softwareupdate/__init__.py index ad1683a1d..399a70a19 100644 --- a/src/octoprint/plugins/softwareupdate/__init__.py +++ b/src/octoprint/plugins/softwareupdate/__init__.py @@ -400,14 +400,13 @@ def get_current_versions(self, check_targets=None, force=False): if not target in check_targets: continue - populated_check = self._populated_check(target, check) - try: + populated_check = self._populated_check(target, check) target_information, target_update_available, target_update_possible = self._get_current_version(target, populated_check, force=force) if target_information is None: target_information = dict() except exceptions.UnknownCheckType: - self._logger.warn("Unknown update check type for %s" % target) + self._logger.warn("Unknown update check type for target {}".format(target)) continue target_information = dict_merge(dict(local=dict(name="unknown", value="unknown"), remote=dict(name="unknown", value="unknown")), target_information) @@ -655,6 +654,9 @@ def _perform_restart(self, restart_command): raise exceptions.RestartFailed() def _populated_check(self, target, check): + if not "type" in check: + raise exceptions.UnknownCheckType() + result = dict(check) if target == "octoprint": From 43ca4d8252eda987d1be6bad8b6531558e9f6974 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Mon, 21 Sep 2015 16:42:11 +0200 Subject: [PATCH 05/34] SWU: Do not overwrite check information again Current version information of OctoPrint from a check definition could be overwritten for checks under certain circumstances. --- src/octoprint/plugins/softwareupdate/__init__.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/octoprint/plugins/softwareupdate/__init__.py b/src/octoprint/plugins/softwareupdate/__init__.py index 399a70a19..bb324b37b 100644 --- a/src/octoprint/plugins/softwareupdate/__init__.py +++ b/src/octoprint/plugins/softwareupdate/__init__.py @@ -689,13 +689,6 @@ def _get_version_checker(self, target, check): if not "type" in check: raise exceptions.ConfigurationInvalid("no check type defined") - if target == "octoprint": - from octoprint._version import get_versions - from flask.ext.babel import gettext - check["displayName"] = gettext("OctoPrint") - check["displayVersion"] = "{octoprint_version}" - check["current"] = get_versions()["version"] - check_type = check["type"] if check_type == "github_release": return version_checks.github_release From 25a4d4b79b3f3f24a95c475fc56d1cf43324d17a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Sat, 12 Sep 2015 11:09:28 +0200 Subject: [PATCH 06/34] SWU: Track check origins, ignore if from unavailable plugin There was a problem with software update checks configurations stored in config.yaml for which the providing plugin was then removed, since those check definitions then lacked their default values to be merged on whatever was stored in config.yaml, causing incomplete check configurations as a consequence over which the plugin tripped. This patch fixes that in that it tracks which check config keys are provided by plugins and only returns those as the active check configurations that belong to plugins that are still in the system. TODO: This is only half of the solution. Check configurations of plugins that are being uninstalled should be removed from the config if the user decides to remove any settings by the plugin too. We need some adjustments in the lifecycle tracking in order to make this possible however, so for now this must suffice to at least prevent any errors from occuring when incomplete configs are encountered. (cherry picked from commit 8af8b8f) --- .../plugins/softwareupdate/__init__.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/octoprint/plugins/softwareupdate/__init__.py b/src/octoprint/plugins/softwareupdate/__init__.py index bb324b37b..a3f7d2ef7 100644 --- a/src/octoprint/plugins/softwareupdate/__init__.py +++ b/src/octoprint/plugins/softwareupdate/__init__.py @@ -59,6 +59,7 @@ def _get_configured_checks(self): self._refresh_configured_checks = False self._configured_checks = self._settings.get(["checks"], merged=True) update_check_hooks = self._plugin_manager.get_hooks("octoprint.plugin.softwareupdate.check_config") + check_providers = self._settings.get(["check_providers"], merged=True) for name, hook in update_check_hooks.items(): try: hook_checks = hook() @@ -66,9 +67,23 @@ def _get_configured_checks(self): self._logger.exception("Error while retrieving update information from plugin {name}".format(**locals())) else: for key, data in hook_checks.items(): + check_providers[key] = name if key in self._configured_checks: data = dict_merge(data, self._configured_checks[key]) self._configured_checks[key] = data + self._settings.set(["check_providers"], check_providers) + self._settings.save() + + # we only want to process checks that came from plugins for + # which the plugins are still installed and enabled + config_checks = self._settings.get(["checks"]) + plugin_and_not_enabled = lambda k: k in check_providers and \ + not check_providers[k] in self._plugin_manager.enabled_plugins + obsolete_plugin_checks = filter(plugin_and_not_enabled, + config_checks.keys()) + for key in obsolete_plugin_checks: + self._logger.debug("Check for key {} was provided by plugin {} that's no longer available, ignoring it".format(key, check_providers[key])) + del self._configured_checks[key] return self._configured_checks @@ -134,6 +149,7 @@ def get_settings_defaults(self): }, }, "pip_command": None, + "check_providers": {}, "cache_ttl": 24 * 60, } @@ -406,7 +422,7 @@ def get_current_versions(self, check_targets=None, force=False): if target_information is None: target_information = dict() except exceptions.UnknownCheckType: - self._logger.warn("Unknown update check type for target {}".format(target)) + self._logger.warn("Unknown update check type for target {}: {}".format(target, check.get("type", ""))) continue target_information = dict_merge(dict(local=dict(name="unknown", value="unknown"), remote=dict(name="unknown", value="unknown")), target_information) From c26515c13d52ab290a275eba4ca492de0dcd8408 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Tue, 22 Sep 2015 11:35:47 +0200 Subject: [PATCH 07/34] PipCaller: Allow update of used pip command --- src/octoprint/util/pip.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/octoprint/util/pip.py b/src/octoprint/util/pip.py index 980e5735c..94b139267 100644 --- a/src/octoprint/util/pip.py +++ b/src/octoprint/util/pip.py @@ -21,14 +21,13 @@ class PipCaller(object): def __init__(self, configured=None): self._logger = logging.getLogger(__name__) - self._configured = configured + self.configured = configured self._command = None self._version = None - self._command, self._version = self._find_pip() + self.trigger_refresh() - self.refresh = False self.on_log_call = lambda *args, **kwargs: None self.on_log_stdout = lambda *args, **kwargs: None self.on_log_stderr = lambda *args, **kwargs: None @@ -57,10 +56,18 @@ def version(self): def available(self): return self._command is not None + def trigger_refresh(self): + try: + self._command, self._version = self._find_pip() + except: + self._logger.exception("Error while discovering pip command") + self._command = None + self._version = None + self.refresh = False + def execute(self, *args): if self.refresh: - self._command, self._version = self._find_pip() - self.refresh = False + self.trigger_refresh() if self._command is None: raise UnknownPip() @@ -111,7 +118,7 @@ def execute(self, *args): def _find_pip(self): - pip_command = self._configured + pip_command = self.configured pip_version = None if pip_command is None: From 65bc28a03e56275174918f4b25c4c035d84408aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Tue, 22 Sep 2015 11:36:57 +0200 Subject: [PATCH 08/34] PMGR: Added configuration dialog and info re used pip binary & version --- .../plugins/pluginmanager/__init__.py | 26 +++++- .../pluginmanager/static/js/pluginmanager.js | 81 +++++++++++++++++-- .../templates/pluginmanager_settings.jinja2 | 56 +++++++++++++ 3 files changed, 155 insertions(+), 8 deletions(-) diff --git a/src/octoprint/plugins/pluginmanager/__init__.py b/src/octoprint/plugins/pluginmanager/__init__.py index c7950f481..8390cd8e2 100644 --- a/src/octoprint/plugins/pluginmanager/__init__.py +++ b/src/octoprint/plugins/pluginmanager/__init__.py @@ -87,9 +87,17 @@ def get_settings_defaults(self): ) def on_settings_save(self, data): + old_pip = self._settings.get(["pip"]) octoprint.plugin.SettingsPlugin.on_settings_save(self, data) + new_pip = self._settings.get(["pip"]) + self._repository_cache_ttl = self._settings.get_int(["repository_ttl"]) * 60 - self._pip_caller.refresh = True + if old_pip != new_pip: + self._pip_caller.configured = new_pip + try: + self._pip_caller.trigger_refresh() + except: + self._pip_caller ##~~ AssetPlugin @@ -169,7 +177,18 @@ def on_api_get(self, request): if "refresh_repository" in request.values and request.values["refresh_repository"] in valid_boolean_trues: self._repository_available = self._refresh_repository() - return jsonify(plugins=result, repository=dict(available=self._repository_available, plugins=self._repository_plugins), os=self._get_os(), octoprint=self._get_octoprint_version()) + return jsonify(plugins=result, + repository=dict( + available=self._repository_available, + plugins=self._repository_plugins + ), + os=self._get_os(), + octoprint=self._get_octoprint_version(), + pip=dict( + available=self._pip_caller.available, + command=self._pip_caller.command, + version=str(self._pip_caller.version) + )) def on_api_command(self, command, data): if not admin_permission.can(): @@ -603,7 +622,8 @@ def _to_external_representation(self, plugin): pending_enable=(not plugin.enabled and plugin.key in self._pending_enable), pending_disable=(plugin.enabled and plugin.key in self._pending_disable), pending_install=(plugin.key in self._pending_install), - pending_uninstall=(plugin.key in self._pending_uninstall) + pending_uninstall=(plugin.key in self._pending_uninstall), + origin=plugin.origin.type ) __plugin_name__ = "Plugin Manager" diff --git a/src/octoprint/plugins/pluginmanager/static/js/pluginmanager.js b/src/octoprint/plugins/pluginmanager/static/js/pluginmanager.js index 7c5265ff5..35a634d8b 100644 --- a/src/octoprint/plugins/pluginmanager/static/js/pluginmanager.js +++ b/src/octoprint/plugins/pluginmanager/static/js/pluginmanager.js @@ -6,6 +6,12 @@ $(function() { self.settingsViewModel = parameters[1]; self.printerState = parameters[2]; + self.config_repositoryUrl = ko.observable(); + self.config_repositoryTtl = ko.observable(); + self.config_pipCommand = ko.observable(); + + self.configurationDialog = $("#settings_plugin_pluginmanager_configurationdialog"); + self.plugins = new ItemListHelper( "plugin.pluginmanager.installedplugins", { @@ -72,6 +78,10 @@ $(function() { self.followDependencyLinks = ko.observable(false); + self.pipAvailable = ko.observable(false); + self.pipCommand = ko.observable(); + self.pipVersion = ko.observable(); + self.working = ko.observable(false); self.workingTitle = ko.observable(); self.workingDialog = undefined; @@ -86,11 +96,15 @@ $(function() { }; self.enableUninstall = function(data) { - return self.enableManagement() && !data.bundled && data.key != 'pluginmanager' && !data.pending_uninstall; + return self.enableManagement() + && (data.origin != "entry_point" || self.pipAvailable()) + && !data.bundled + && data.key != 'pluginmanager' + && !data.pending_uninstall; }; self.enableRepoInstall = function(data) { - return self.enableManagement() && self.isCompatible(data); + return self.enableManagement() && self.pipAvailable() && self.isCompatible(data); }; self.invalidUrl = ko.computed(function() { @@ -100,7 +114,7 @@ $(function() { self.enableUrlInstall = ko.computed(function() { var url = self.installUrl(); - return self.enableManagement() && url !== undefined && url.trim() != "" && !self.invalidUrl(); + return self.enableManagement() && self.pipAvailable() && url !== undefined && url.trim() != "" && !self.invalidUrl(); }); self.invalidArchive = ko.computed(function() { @@ -110,7 +124,7 @@ $(function() { self.enableArchiveInstall = ko.computed(function() { var name = self.uploadFilename(); - return self.enableManagement() && name !== undefined && name.trim() != "" && !self.invalidArchive(); + return self.enableManagement() && self.pipAvailable() && name !== undefined && name.trim() != "" && !self.invalidArchive(); }); self.uploadElement.fileupload({ @@ -167,7 +181,8 @@ $(function() { self.fromResponse = function(data) { self._fromPluginsResponse(data.plugins); - self._fromRepositoryResponse(data.repository) + self._fromRepositoryResponse(data.repository); + self._fromPipResponse(data.pip); }; self._fromPluginsResponse = function(data) { @@ -188,6 +203,17 @@ $(function() { } }; + self._fromPipResponse = function(data) { + self.pipAvailable(data.available); + if (data.available) { + self.pipCommand(data.command); + self.pipVersion(data.version); + } else { + self.pipCommand(undefined); + self.pipVersion(undefined); + } + }; + self.requestData = function(includeRepo) { if (!self.loginState.isAdmin()) { return; @@ -343,6 +369,51 @@ $(function() { self.requestData(true); }; + self.showPluginSettings = function() { + self._copyConfig(); + self.configurationDialog.modal(); + }; + + self.savePluginSettings = function() { + var pipCommand = self.config_pipCommand(); + if (pipCommand != undefined && pipCommand.trim() == "") { + pipCommand = null; + } + + var repository = self.config_repositoryUrl(); + if (repository != undefined && repository.trim() == "") { + repository = null; + } + + var repositoryTtl; + try { + repositoryTtl = parseInt(self.config_repositoryTtl()); + } catch (ex) { + repositoryTtl = null; + } + + var data = { + plugins: { + pluginmanager: { + repository: repository, + repository_ttl: repositoryTtl, + pip: pipCommand + } + } + }; + self.settingsViewModel.saveData(data, function() { + self.configurationDialog.modal("hide"); + self._copyConfig(); + self.refreshRepository(); + }); + }; + + self._copyConfig = function() { + self.config_repositoryUrl(self.settingsViewModel.settings.plugins.pluginmanager.repository()); + self.config_repositoryTtl(self.settingsViewModel.settings.plugins.pluginmanager.repository_ttl()); + self.config_pipCommand(self.settingsViewModel.settings.plugins.pluginmanager.pip()); + }; + self.installed = function(data) { return _.includes(self.installedPlugins(), data.id); }; diff --git a/src/octoprint/plugins/pluginmanager/templates/pluginmanager_settings.jinja2 b/src/octoprint/plugins/pluginmanager/templates/pluginmanager_settings.jinja2 index b855dc81f..7c00534a3 100644 --- a/src/octoprint/plugins/pluginmanager/templates/pluginmanager_settings.jinja2 +++ b/src/octoprint/plugins/pluginmanager/templates/pluginmanager_settings.jinja2 @@ -4,7 +4,20 @@ {% endmacro %} +{% macro pluginmanager_nopip() %} +
{% trans %} + The pip command could not be detected automatically, + please configure it manually. No installation and uninstallation of plugin + packages is possible while pip is unavailable. +{% endtrans %}
+{% endmacro %} + {{ pluginmanager_printing() }} +{{ pluginmanager_nopip() }} + +
+ +

{{ _('Installed Plugins') }}

@@ -47,6 +60,10 @@ +

+ Using pip at "" (Version ) +

+ + + + From fd4271a962b8e4b3fb87eb65ceb7bceb48a63a27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Tue, 22 Sep 2015 11:39:45 +0200 Subject: [PATCH 09/34] PipCaller: Added back missing member variable --- src/octoprint/util/pip.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/octoprint/util/pip.py b/src/octoprint/util/pip.py index 94b139267..7fa3dbc36 100644 --- a/src/octoprint/util/pip.py +++ b/src/octoprint/util/pip.py @@ -22,6 +22,7 @@ def __init__(self, configured=None): self._logger = logging.getLogger(__name__) self.configured = configured + self.refresh = False self._command = None self._version = None From b4b5689bc47264d7a2e7774abb9fd8e9083be670 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Tue, 22 Sep 2015 15:23:50 +0200 Subject: [PATCH 10/34] Fix: Correctly handle unset Content-Type header for command requests --- src/octoprint/server/util/flask.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/octoprint/server/util/flask.py b/src/octoprint/server/util/flask.py index b67a09080..7fe8ae34a 100644 --- a/src/octoprint/server/util/flask.py +++ b/src/octoprint/server/util/flask.py @@ -530,7 +530,8 @@ def get_remote_address(request): def get_json_command_from_request(request, valid_commands): - if not "application/json" in request.headers["Content-Type"]: + content_type = request.headers.get("Content-Type", None) + if content_type is None or not "application/json" in content_type: return None, None, make_response("Expected content-type JSON", 400) data = request.json From 1b4ea75466d84b3e96cc19d12e40e71db1d7cad2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Mon, 28 Sep 2015 12:55:32 +0200 Subject: [PATCH 11/34] Fix: Don't have file upload widgets listen to drop events by default That way they won't be triggered by gcode uploads when all they are interested in are uploads via a single file input. --- src/octoprint/static/js/app/main.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/octoprint/static/js/app/main.js b/src/octoprint/static/js/app/main.js index 56617dbae..bab804bb9 100644 --- a/src/octoprint/static/js/app/main.js +++ b/src/octoprint/static/js/app/main.js @@ -21,6 +21,15 @@ $(function() { headers: {"X-Api-Key": UI_API_KEY} }); + //~~ Initialize file upload plugin + + $.widget("blueimp.fileupload", $.blueimp.fileupload, { + options: { + dropZone: null, + pasteZone: null + } + }); + //~~ Initialize i18n var catalog = window["BABEL_TO_LOAD_" + LOCALE]; From 9ff5c363707b12086119eb33ff7c271592deaa75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Mon, 28 Sep 2015 13:08:35 +0200 Subject: [PATCH 12/34] Fixed a documentation bug --- docs/api/logs.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/api/logs.rst b/docs/api/logs.rst index b2759aadb..875427842 100644 --- a/docs/api/logs.rst +++ b/docs/api/logs.rst @@ -15,7 +15,7 @@ Log file management Retrieve a list of available log files ====================================== -.. http:post:: /api/logs +.. http:get:: /api/logs Retrieve information regarding all log files currently available and regarding the disk space still available in the system on the location the log files are being stored. From 1178fe9e95a77efafcbfbcefa94836cb1d0199f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Mon, 28 Sep 2015 19:53:30 +0200 Subject: [PATCH 13/34] Support sudo for installing plugins, but warn about it --- README.md | 16 ++++++++++++- .../plugins/pluginmanager/__init__.py | 9 ++++++- .../pluginmanager/static/js/pluginmanager.js | 16 ++++++++++++- .../templates/pluginmanager_settings.jinja2 | 24 ++++++++++++++++--- src/octoprint/util/pip.py | 22 ++++++++++++++--- 5 files changed, 78 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 06d5b6b04..4d5a9ba0b 100644 --- a/README.md +++ b/README.md @@ -38,11 +38,25 @@ For information about how to go about contributions of any kind, please see the Installation ------------ -Installation instructions for installing from source for different operating systems can be found [on the wiki](https://github.com/foosel/OctoPrint/wiki#assorted-guides). +Installation instructions for installing from source for different operating +systems can be found [on the wiki](https://github.com/foosel/OctoPrint/wiki#assorted-guides). If you want to run OctoPrint on a Raspberry Pi you might want to take a look at [OctoPi](https://github.com/guysoft/OctoPi) which is a custom SD card image that includes OctoPrint plus dependencies. +The generic steps that should basically be done regardless of operating system +and runtime environment are the following (as *regular +user*, please keep your hands *off* of the `sudo` command here!) - this assumes +you already have Python 2.7, pip and virtualenv set up: + +1. Checkout OctoPrint: `git clone https://github.com/foosel/OctoPrint.git` +2. Change into the OctoPrint folder: `cd OctoPrint` +3. Create a user-owned virtual environment therein: `virtualenv --system-site-packages venv` +4. Install OctoPrint *into that virtual environment*: `./venv/bin/python setup.py install` + +You may then start the OctoPrint server via `/path/to/OctoPrint/venv/bin/octoprint`, see [Usage](#usage) +for details. + After installation, please make sure you follow the first-run wizard and set up access control as necessary. If you want to not only be notified about new releases but also be able to automatically upgrade to them from within diff --git a/src/octoprint/plugins/pluginmanager/__init__.py b/src/octoprint/plugins/pluginmanager/__init__.py index 8390cd8e2..a92fa91c1 100644 --- a/src/octoprint/plugins/pluginmanager/__init__.py +++ b/src/octoprint/plugins/pluginmanager/__init__.py @@ -82,6 +82,7 @@ def get_settings_defaults(self): repository="http://plugins.octoprint.org/plugins.json", repository_ttl=24*60, pip=None, + pip_args=None, dependency_links=False, hidden=[] ) @@ -187,7 +188,9 @@ def on_api_get(self, request): pip=dict( available=self._pip_caller.available, command=self._pip_caller.command, - version=str(self._pip_caller.version) + version=str(self._pip_caller.version), + use_sudo=self._pip_caller.use_sudo, + additional_args=self._settings.get(["pip_args"]) )) def on_api_command(self, command, data): @@ -447,6 +450,10 @@ def _call_pip(self, args): if self._pip_caller < self._pip_version_dependency_links: args.remove("--process-dependency-links") + additional_args = self._settings.get(["pip_args"]) + if additional_args: + args.append(additional_args) + return self._pip_caller.execute(*args) def _log_message(self, *lines): diff --git a/src/octoprint/plugins/pluginmanager/static/js/pluginmanager.js b/src/octoprint/plugins/pluginmanager/static/js/pluginmanager.js index 35a634d8b..75c8e2f39 100644 --- a/src/octoprint/plugins/pluginmanager/static/js/pluginmanager.js +++ b/src/octoprint/plugins/pluginmanager/static/js/pluginmanager.js @@ -9,6 +9,7 @@ $(function() { self.config_repositoryUrl = ko.observable(); self.config_repositoryTtl = ko.observable(); self.config_pipCommand = ko.observable(); + self.config_pipAdditionalArgs = ko.observable(); self.configurationDialog = $("#settings_plugin_pluginmanager_configurationdialog"); @@ -81,6 +82,8 @@ $(function() { self.pipAvailable = ko.observable(false); self.pipCommand = ko.observable(); self.pipVersion = ko.observable(); + self.pipUseSudo = ko.observable(); + self.pipAdditionalArgs = ko.observable(); self.working = ko.observable(false); self.workingTitle = ko.observable(); @@ -208,9 +211,13 @@ $(function() { if (data.available) { self.pipCommand(data.command); self.pipVersion(data.version); + self.pipUseSudo(data.use_sudo); + self.pipAdditionalArgs(data.additional_args); } else { self.pipCommand(undefined); self.pipVersion(undefined); + self.pipUseSudo(undefined); + self.pipAdditionalArgs(undefined); } }; @@ -392,12 +399,18 @@ $(function() { repositoryTtl = null; } + var pipArgs = self.config_pipAdditionalArgs(); + if (pipArgs != undefined && pipArgs.trim() == "") { + pipArgs = null; + } + var data = { plugins: { pluginmanager: { repository: repository, repository_ttl: repositoryTtl, - pip: pipCommand + pip: pipCommand, + pip_args: pipArgs } } }; @@ -412,6 +425,7 @@ $(function() { self.config_repositoryUrl(self.settingsViewModel.settings.plugins.pluginmanager.repository()); self.config_repositoryTtl(self.settingsViewModel.settings.plugins.pluginmanager.repository_ttl()); self.config_pipCommand(self.settingsViewModel.settings.plugins.pluginmanager.pip()); + self.config_pipAdditionalArgs(self.settingsViewModel.settings.plugins.pluginmanager.pip_args()); }; self.installed = function(data) { diff --git a/src/octoprint/plugins/pluginmanager/templates/pluginmanager_settings.jinja2 b/src/octoprint/plugins/pluginmanager/templates/pluginmanager_settings.jinja2 index 7c00534a3..d399b593c 100644 --- a/src/octoprint/plugins/pluginmanager/templates/pluginmanager_settings.jinja2 +++ b/src/octoprint/plugins/pluginmanager/templates/pluginmanager_settings.jinja2 @@ -6,14 +6,25 @@ {% macro pluginmanager_nopip() %}
{% trans %} - The pip command could not be detected automatically, - please configure it manually. No installation and uninstallation of plugin + The pip command could not be found. + Please configure it manually. No installation and uninstallation of plugin packages is possible while pip is unavailable. {% endtrans %}
{% endmacro %} +{% macro pluginmanager_sudopip() %} +
{% trans %} + The pip command is configured to use sudo. This + is not recommended due to security reasons. It is strongly + suggested you install OctoPrint under a + user-owned virtual environment + so that the use of sudo is not needed for plugin management. +{% endtrans %}
+{% endmacro %} + {{ pluginmanager_printing() }} {{ pluginmanager_nopip() }} +{{ pluginmanager_sudopip() }}
@@ -61,7 +72,7 @@

- Using pip at "" (Version ) + Using pip at "" (Version , additional arguments: )

+
+ +
+ +
+
diff --git a/src/octoprint/util/pip.py b/src/octoprint/util/pip.py index 7fa3dbc36..11c6f1189 100644 --- a/src/octoprint/util/pip.py +++ b/src/octoprint/util/pip.py @@ -26,6 +26,7 @@ def __init__(self, configured=None): self._command = None self._version = None + self._use_sudo = False self.trigger_refresh() @@ -53,13 +54,17 @@ def command(self): def version(self): return self._version + @property + def use_sudo(self): + return self._use_sudo + @property def available(self): return self._command is not None def trigger_refresh(self): try: - self._command, self._version = self._find_pip() + self._command, self._version, self._use_sudo = self._find_pip() except: self._logger.exception("Error while discovering pip command") self._command = None @@ -74,6 +79,8 @@ def execute(self, *args): raise UnknownPip() command = [self._command] + list(args) + if self._use_sudo: + command = ["sudo"] + command joined_command = " ".join(command) self._logger.debug(u"Calling: {}".format(joined_command)) @@ -120,6 +127,11 @@ def execute(self, *args): def _find_pip(self): pip_command = self.configured + if pip_command is not None and pip_command.startswith("sudo "): + pip_command = pip_command[len("sudo "):] + pip_sudo = True + else: + pip_sudo = False pip_version = None if pip_command is None: @@ -152,7 +164,11 @@ def _find_pip(self): if pip_command is not None: self._logger.debug("Found pip at {}, going to figure out its version".format(pip_command)) - p = sarge.run([pip_command, "--version"], stdout=sarge.Capture(), stderr=sarge.Capture()) + + sarge_command = [pip_command, "--version"] + if pip_sudo: + sarge_command = ["sudo"] + sarge_command + p = sarge.run(sarge_command, stdout=sarge.Capture(), stderr=sarge.Capture()) if p.returncode != 0: self._logger.warn("Error while trying to run pip --version: {}".format(p.stderr.text)) @@ -182,7 +198,7 @@ def _find_pip(self): else: self._logger.info("Found pip at {}, version is {}".format(pip_command, version_segment)) - return pip_command, pip_version + return pip_command, pip_version, pip_sudo def _log_stdout(self, *lines): self.on_log_stdout(*lines) From fce7b40b513fdfed82f62f1d364899546ff4b6c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Mon, 28 Sep 2015 20:20:56 +0200 Subject: [PATCH 14/34] pip: Use string representation of version for display in UI --- src/octoprint/plugins/pluginmanager/__init__.py | 2 +- src/octoprint/util/pip.py | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/octoprint/plugins/pluginmanager/__init__.py b/src/octoprint/plugins/pluginmanager/__init__.py index a92fa91c1..a93b6cc27 100644 --- a/src/octoprint/plugins/pluginmanager/__init__.py +++ b/src/octoprint/plugins/pluginmanager/__init__.py @@ -188,7 +188,7 @@ def on_api_get(self, request): pip=dict( available=self._pip_caller.available, command=self._pip_caller.command, - version=str(self._pip_caller.version), + version=self._pip_caller.version_string, use_sudo=self._pip_caller.use_sudo, additional_args=self._settings.get(["pip_args"]) )) diff --git a/src/octoprint/util/pip.py b/src/octoprint/util/pip.py index 11c6f1189..60ceb1b68 100644 --- a/src/octoprint/util/pip.py +++ b/src/octoprint/util/pip.py @@ -26,6 +26,7 @@ def __init__(self, configured=None): self._command = None self._version = None + self._version_string = None self._use_sudo = False self.trigger_refresh() @@ -54,6 +55,10 @@ def command(self): def version(self): return self._version + @property + def version_string(self): + return self._version_string + @property def use_sudo(self): return self._use_sudo @@ -64,7 +69,7 @@ def available(self): def trigger_refresh(self): try: - self._command, self._version, self._use_sudo = self._find_pip() + self._command, self._version, self._version_string, self._use_sudo = self._find_pip() except: self._logger.exception("Error while discovering pip command") self._command = None @@ -127,12 +132,15 @@ def execute(self, *args): def _find_pip(self): pip_command = self.configured + if pip_command is not None and pip_command.startswith("sudo "): pip_command = pip_command[len("sudo "):] pip_sudo = True else: pip_sudo = False + pip_version = None + version_segment = None if pip_command is None: import os @@ -198,7 +206,7 @@ def _find_pip(self): else: self._logger.info("Found pip at {}, version is {}".format(pip_command, version_segment)) - return pip_command, pip_version, pip_sudo + return pip_command, pip_version, version_segment, pip_sudo def _log_stdout(self, *lines): self.on_log_stdout(*lines) From 5cc8ec5cc39913a7ed11a5a156c91a3014ce6757 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Wed, 30 Sep 2015 16:00:32 +0200 Subject: [PATCH 15/34] Better wording for plugin system startup & sorted plugin list of ALL plugins --- src/octoprint/plugin/core.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/octoprint/plugin/core.py b/src/octoprint/plugin/core.py index 913289de8..bcb28ce01 100644 --- a/src/octoprint/plugin/core.py +++ b/src/octoprint/plugin/core.py @@ -861,7 +861,7 @@ def initialize_implementations(self, additional_injects=None, additional_inject_ additional_pre_inits=additional_pre_inits, additional_post_inits=additional_post_inits) - self.logger.info("Initialized {count} plugin(s)".format(count=len(self.plugin_implementations))) + self.logger.info("Initialized {count} plugin implementation(s)".format(count=len(self.plugin_implementations))) def initialize_implementation_of_plugin(self, name, plugin, additional_injects=None, additional_inject_factories=None, additional_pre_inits=None, additional_post_inits=None): if plugin.implementation is None: @@ -955,15 +955,13 @@ def log_all_plugins(self, show_bundled=True, bundled_str=(" (bundled)", ""), sho self.logger.info("No plugins available") else: self.logger.info("{count} plugin(s) registered with the system:\n{plugins}".format(count=len(all_plugins), plugins="\n".join( - sorted( - map(lambda x: "| " + x.long_str(show_bundled=show_bundled, - bundled_strs=bundled_str, - show_location=show_location, - location_str=location_str, - show_enabled=show_enabled, - enabled_strs=enabled_str), - self.enabled_plugins.values()) - ) + map(lambda x: "| " + x.long_str(show_bundled=show_bundled, + bundled_strs=bundled_str, + show_location=show_location, + location_str=location_str, + show_enabled=show_enabled, + enabled_strs=enabled_str), + sorted(self.plugins.values(), key=lambda x: str(x).lower())) ))) def get_plugin(self, identifier, require_enabled=True): From 918ffa2557d9bde56e7c3a50e66ac40210796658 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Mon, 20 Jul 2015 16:42:28 +0200 Subject: [PATCH 16/34] Refactored timelapse core Capturing is now queue based, rendering will not start until all images have been captured, and timed postroll does not depend on system time anymore. Also refactored some of the names to be python naming compliant while at it. (cherry picked from commit 4f5dc70) --- src/octoprint/events.py | 3 + src/octoprint/server/api/timelapse.py | 10 +- src/octoprint/static/js/app/dataupdater.js | 19 +- src/octoprint/timelapse.py | 330 ++++++++++++--------- 4 files changed, 224 insertions(+), 138 deletions(-) diff --git a/src/octoprint/events.py b/src/octoprint/events.py index 828862336..2cc765d9b 100644 --- a/src/octoprint/events.py +++ b/src/octoprint/events.py @@ -74,6 +74,9 @@ class Events(object): # Timelapse CAPTURE_START = "CaptureStart" CAPTURE_DONE = "CaptureDone" + CAPTURE_FAILED = "CaptureFailed" + POSTROLL_START = "PostRollStart" + POSTROLL_END = "PostRollEnd" MOVIE_RENDERING = "MovieRendering" MOVIE_DONE = "MovieDone" MOVIE_FAILED = "MovieFailed" diff --git a/src/octoprint/server/api/timelapse.py b/src/octoprint/server/api/timelapse.py index bef855365..63b5be342 100644 --- a/src/octoprint/server/api/timelapse.py +++ b/src/octoprint/server/api/timelapse.py @@ -29,14 +29,14 @@ def getTimelapseData(): config = {"type": "off"} if timelapse is not None and isinstance(timelapse, octoprint.timelapse.ZTimelapse): config["type"] = "zchange" - config["postRoll"] = timelapse.postRoll() - config["fps"] = timelapse.fps() + config["postRoll"] = timelapse.post_roll + config["fps"] = timelapse.fps elif timelapse is not None and isinstance(timelapse, octoprint.timelapse.TimedTimelapse): config["type"] = "timed" - config["postRoll"] = timelapse.postRoll() - config["fps"] = timelapse.fps() + config["postRoll"] = timelapse.post_roll + config["fps"] = timelapse.fps config.update({ - "interval": timelapse.interval() + "interval": timelapse.interval }) files = octoprint.timelapse.getFinishedTimelapses() diff --git a/src/octoprint/static/js/app/dataupdater.js b/src/octoprint/static/js/app/dataupdater.js index 53f5ef705..6a63cd28b 100644 --- a/src/octoprint/static/js/app/dataupdater.js +++ b/src/octoprint/static/js/app/dataupdater.js @@ -177,6 +177,7 @@ function DataUpdater(allViewModels) { var type = data["type"]; var payload = data["payload"]; var html = ""; + var format = {}; log.debug("Got event " + type + " with payload: " + JSON.stringify(payload)); @@ -187,7 +188,23 @@ function DataUpdater(allViewModels) { } else if (type == "MovieFailed") { html = "

" + _.sprintf(gettext("Rendering of timelapse %(movie_basename)s failed with return code %(returncode)s"), payload) + "

"; html += pnotifyAdditionalInfo('
' + payload.error + '
'); - new PNotify({title: gettext("Rendering failed"), text: html, type: "error", hide: false}); + new PNotify({ + title: gettext("Rendering failed"), + text: html, + type: "error", + hide: false + }); + } else if (type == "PostRollStart") { + if (payload.postroll_duration > 60) { + format = {duration: _.sprintf(gettext("%(minutes)d min"), {minutes: payload.postroll_duration / 60})}; + } else { + format = {duration: _.sprintf(gettext("%(seconds)d sec"), {seconds: payload.postroll_duration})}; + } + + new PNotify({ + title: gettext("Capturing timelapse postroll"), + text: _.sprintf(gettext("Now capturing timelapse post roll, this will take approximately %(duration)s..."), format) + }); } else if (type == "SlicingStarted") { gcodeUploadProgress.addClass("progress-striped").addClass("active"); gcodeUploadProgressBar.css("width", "100%"); diff --git a/src/octoprint/timelapse.py b/src/octoprint/timelapse.py index e1e0b87f9..f4862b73a 100644 --- a/src/octoprint/timelapse.py +++ b/src/octoprint/timelapse.py @@ -6,13 +6,13 @@ import logging import os import threading -import urllib import time -import subprocess import fnmatch import datetime import sys import shutil +import Queue +import requests import octoprint.util as util @@ -58,7 +58,7 @@ def notifyCallbacks(timelapse): if timelapse is None: config = None else: - config = timelapse.configData() + config = timelapse.config_data() for callback in updateCallbacks: try: callback.sendTimelapseConfig(config) except: logging.getLogger(__name__).exception("Exception while pushing timelapse configuration") @@ -86,12 +86,12 @@ def configureTimelapse(config=None, persist=False): if type is None or "off" == type: current = None elif "zchange" == type: - current = ZTimelapse(postRoll=postRoll, fps=fps) + current = ZTimelapse(post_roll=postRoll, fps=fps) elif "timed" == type: interval = 10 if "options" in config and "interval" in config["options"] and config["options"]["interval"] > 0: interval = config["options"]["interval"] - current = TimedTimelapse(postRoll=postRoll, interval=interval, fps=fps) + current = TimedTimelapse(post_roll=postRoll, interval=interval, fps=fps) notifyCallbacks(current) @@ -101,72 +101,83 @@ def configureTimelapse(config=None, persist=False): class Timelapse(object): - def __init__(self, postRoll=0, fps=25): + QUEUE_ENTRY_TYPE_CAPTURE = "capture" + QUEUE_ENTRY_TYPE_CALLBACK = "callback" + + def __init__(self, post_roll=0, fps=25): self._logger = logging.getLogger(__name__) - self._imageNumber = None - self._inTimelapse = False - self._gcodeFile = None + self._image_number = None + self._in_timelapse = False + self._gcode_file = None - self._postRoll = postRoll - self._postRollStart = None - self._onPostRollDone = None + self._post_roll = post_roll + self._on_post_roll_done = None - self._captureDir = settings().getBaseFolder("timelapse_tmp") - self._movieDir = settings().getBaseFolder("timelapse") - self._snapshotUrl = settings().get(["webcam", "snapshot"]) - self._ffmpegThreads = settings().get(["webcam", "ffmpegThreads"]) + self._capture_dir = settings().getBaseFolder("timelapse_tmp") + self._movie_dir = settings().getBaseFolder("timelapse") + self._snapshot_url = settings().get(["webcam", "snapshot"]) + self._ffmpeg_threads = settings().get(["webcam", "ffmpegThreads"]) self._fps = fps - self._renderThread = None - self._captureMutex = threading.Lock() + self._render_thread = None + + self._capture_mutex = threading.Lock() + self._capture_queue = Queue.Queue() + self._capture_queue_active = True + + self._capture_queue_thread = threading.Thread(target=self._capture_queue_worker) + self._capture_queue_thread.daemon = True + self._capture_queue_thread.start() # subscribe events - eventManager().subscribe(Events.PRINT_STARTED, self.onPrintStarted) - eventManager().subscribe(Events.PRINT_FAILED, self.onPrintDone) - eventManager().subscribe(Events.PRINT_DONE, self.onPrintDone) - eventManager().subscribe(Events.PRINT_RESUMED, self.onPrintResumed) - for (event, callback) in self.eventSubscriptions(): + eventManager().subscribe(Events.PRINT_STARTED, self.on_print_started) + eventManager().subscribe(Events.PRINT_FAILED, self.on_print_done) + eventManager().subscribe(Events.PRINT_DONE, self.on_print_done) + eventManager().subscribe(Events.PRINT_RESUMED, self.on_print_resumed) + for (event, callback) in self.event_subscriptions(): eventManager().subscribe(event, callback) - def postRoll(self): - return self._postRoll + @property + def post_roll(self): + return self._post_roll + @property def fps(self): return self._fps def unload(self): - if self._inTimelapse: - self.stopTimelapse(doCreateMovie=False) + if self._in_timelapse: + self.stop_timelapse(doCreateMovie=False) # unsubscribe events - eventManager().unsubscribe(Events.PRINT_STARTED, self.onPrintStarted) - eventManager().unsubscribe(Events.PRINT_FAILED, self.onPrintDone) - eventManager().unsubscribe(Events.PRINT_DONE, self.onPrintDone) - eventManager().unsubscribe(Events.PRINT_RESUMED, self.onPrintResumed) - for (event, callback) in self.eventSubscriptions(): + eventManager().unsubscribe(Events.PRINT_STARTED, self.on_print_started) + eventManager().unsubscribe(Events.PRINT_FAILED, self.on_print_done) + eventManager().unsubscribe(Events.PRINT_DONE, self.on_print_done) + eventManager().unsubscribe(Events.PRINT_RESUMED, self.on_print_resumed) + for (event, callback) in self.event_subscriptions(): eventManager().unsubscribe(event, callback) - def onPrintStarted(self, event, payload): + def on_print_started(self, event, payload): """ Override this to perform additional actions upon start of a print job. """ - self.startTimelapse(payload["file"]) + self.start_timelapse(payload["file"]) - def onPrintDone(self, event, payload): + def on_print_done(self, event, payload): """ Override this to perform additional actions upon the stop of a print job. """ - self.stopTimelapse(success=(event==Events.PRINT_DONE)) + self.stop_timelapse(success=(event==Events.PRINT_DONE)) - def onPrintResumed(self, event, payload): + def on_print_resumed(self, event, payload): """ Override this to perform additional actions upon the pausing of a print job. """ - if not self._inTimelapse: - self.startTimelapse(payload["file"]) + if not self._in_timelapse: + self.start_timelapse(payload["file"]) - def eventSubscriptions(self): + def event_subscriptions(self): """ Override this method to subscribe to additional events by returning an array of (event, callback) tuples. @@ -178,7 +189,7 @@ def eventSubscriptions(self): """ return [] - def configData(self): + def config_data(self): """ Override this method to return the current timelapse configuration data. The data should have the following form: @@ -188,94 +199,139 @@ def configData(self): """ return None - def startTimelapse(self, gcodeFile): + def start_timelapse(self, gcodeFile): self._logger.debug("Starting timelapse for %s" % gcodeFile) - self.cleanCaptureDir() + self.clean_capture_dir() - self._imageNumber = 0 - self._inTimelapse = True - self._gcodeFile = os.path.basename(gcodeFile) + self._image_number = 0 + self._in_timelapse = True + self._gcode_file = os.path.basename(gcodeFile) - def stopTimelapse(self, doCreateMovie=True, success=True): + def stop_timelapse(self, doCreateMovie=True, success=True): self._logger.debug("Stopping timelapse") - self._inTimelapse = False + self._in_timelapse = False def resetImageNumber(): - self._imageNumber = None + self._image_number = None def createMovie(): - self._renderThread = threading.Thread(target=self._createMovie, kwargs={"success": success}) - self._renderThread.daemon = True - self._renderThread.start() + self._render_thread = threading.Thread(target=self._create_movie, kwargs={"success": success}) + self._render_thread.daemon = True + self._render_thread.start() def resetAndCreate(): resetImageNumber() createMovie() - if self._postRoll > 0: - self._postRollStart = time.time() + def waitForCaptures(callback): + self._capture_queue.put(dict(type=self.__class__.QUEUE_ENTRY_TYPE_CALLBACK, callback=callback)) + + def getWaitForCaptures(callback): + def f(): + waitForCaptures(callback) + return f + + if self._post_roll > 0: + eventManager().fire(Events.POSTROLL_START, dict(postroll_duration=self.post_roll * self.fps, postroll_length=self.post_roll, postroll_fps=self.fps)) + self._post_roll_start = time.time() if doCreateMovie: - self._onPostRollDone = resetAndCreate + self._on_post_roll_done = getWaitForCaptures(resetAndCreate) else: - self._onPostRollDone = resetImageNumber - self.processPostRoll() + self._on_post_roll_done = resetImageNumber + self.process_post_roll() else: - self._postRollStart = None + self._post_roll_start = None if doCreateMovie: - resetAndCreate() + waitForCaptures(resetAndCreate) else: resetImageNumber() - def processPostRoll(self): - pass + def process_post_roll(self): + self.post_roll_finished() + + def post_roll_finished(self): + if self.post_roll: + eventManager().fire(Events.POSTROLL_END) + if self._on_post_roll_done is not None: + self._on_post_roll_done() def captureImage(self): - if self._captureDir is None: + if self._capture_dir is None: self._logger.warn("Cannot capture image, capture directory is unset") return - if self._imageNumber is None: - self._logger.warn("Cannot capture image, image number is unset") - return + with self._capture_mutex: + if self._image_number is None: + self._logger.warn("Cannot capture image, image number is unset") + return + + filename = os.path.join(self._capture_dir, "tmp_%05d.jpg" % self._image_number) + self._image_number += 1 - with self._captureMutex: - filename = os.path.join(self._captureDir, "tmp_%05d.jpg" % self._imageNumber) - self._imageNumber += 1 self._logger.debug("Capturing image to %s" % filename) - captureThread = threading.Thread(target=self._captureWorker, kwargs={"filename": filename}) - captureThread.daemon = True - captureThread.start() + entry = dict(type=self.__class__.QUEUE_ENTRY_TYPE_CAPTURE, + filename=filename, + onerror=self._on_capture_error) + self._capture_queue.put(entry) return filename - def _captureWorker(self, filename): + def _on_capture_error(self): + with self._capture_mutex: + if self._image_number is not None and self._image_number > 0: + self._image_number -= 1 + + def _capture_queue_worker(self): + while self._capture_queue_active: + entry = self._capture_queue.get(block=True) + + if entry["type"] == self.__class__.QUEUE_ENTRY_TYPE_CAPTURE and "filename" in entry: + filename = entry["filename"] + onerror = entry.pop("onerror", None) + self._perform_capture(filename, onerror=onerror) + + elif entry["type"] == self.__class__.QUEUE_ENTRY_TYPE_CALLBACK and "callback" in entry: + args = entry.pop("args", []) + kwargs = entry.pop("kwargs", dict()) + entry["callback"](*args, **kwargs) + + def _perform_capture(self, filename, onerror=None): eventManager().fire(Events.CAPTURE_START, {"file": filename}) try: - urllib.urlretrieve(self._snapshotUrl, filename) - self._logger.debug("Image %s captured from %s" % (filename, self._snapshotUrl)) + self._logger.debug("Going to capture %s from %s" % (filename, self._snapshot_url)) + r = requests.get(self._snapshot_url, stream=True) + with open (filename, "wb") as f: + for chunk in r.iter_content(chunk_size=1024): + if chunk: + f.write(chunk) + f.flush() + self._logger.debug("Image %s captured from %s" % (filename, self._snapshot_url)) except: - self._logger.exception("Could not capture image %s from %s, decreasing image counter again" % (filename, self._snapshotUrl)) - with self._captureMutex: - if self._imageNumber is not None and self._imageNumber > 0: - self._imageNumber -= 1 - eventManager().fire(Events.CAPTURE_DONE, {"file": filename}) + self._logger.exception("Could not capture image %s from %s" % (filename, self._snapshot_url)) + if callable(onerror): + onerror() + eventManager().fire(Events.CAPTURE_FAILED, {"file": filename}) + return False + else: + eventManager().fire(Events.CAPTURE_DONE, {"file": filename}) + return True - def _createMovie(self, success=True): + def _create_movie(self, success=True): ffmpeg = settings().get(["webcam", "ffmpeg"]) bitrate = settings().get(["webcam", "bitrate"]) if ffmpeg is None or bitrate is None: self._logger.warn("Cannot create movie, path to ffmpeg or desired bitrate is unset") return - input = os.path.join(self._captureDir, "tmp_%05d.jpg") + input = os.path.join(self._capture_dir, "tmp_%05d.jpg") if success: - output = os.path.join(self._movieDir, "%s_%s.mpg" % (os.path.splitext(self._gcodeFile)[0], time.strftime("%Y%m%d%H%M%S"))) + output = os.path.join(self._movie_dir, "%s_%s.mpg" % (os.path.splitext(self._gcode_file)[0], time.strftime("%Y%m%d%H%M%S"))) else: - output = os.path.join(self._movieDir, "%s_%s-failed.mpg" % (os.path.splitext(self._gcodeFile)[0], time.strftime("%Y%m%d%H%M%S"))) + output = os.path.join(self._movie_dir, "%s_%s-failed.mpg" % (os.path.splitext(self._gcode_file)[0], time.strftime("%Y%m%d%H%M%S"))) # prepare ffmpeg command command = [ - ffmpeg, '-framerate', str(self._fps), '-loglevel', 'error', '-i', input, '-vcodec', 'mpeg2video', '-threads', str(self._ffmpegThreads), '-pix_fmt', 'yuv420p', '-r', str(self._fps), '-y', '-b', bitrate, + ffmpeg, '-framerate', str(self._fps), '-loglevel', 'error', '-i', input, '-vcodec', 'mpeg2video', '-threads', str(self._ffmpeg_threads), '-pix_fmt', 'yuv420p', '-r', str(self._fps), '-y', '-b', bitrate, '-f', 'vob'] filters = [] @@ -315,7 +371,7 @@ def _createMovie(self, success=True): # finalize command with output file self._logger.debug("Rendering movie to %s" % output) command.append("\"" + output + "\"") - eventManager().fire(Events.MOVIE_RENDERING, {"gcode": self._gcodeFile, "movie": output, "movie_basename": os.path.basename(output)}) + eventManager().fire(Events.MOVIE_RENDERING, {"gcode": self._gcode_file, "movie": output, "movie_basename": os.path.basename(output)}) command_str = " ".join(command) self._logger.debug("Executing command: %s" % command_str) @@ -323,75 +379,74 @@ def _createMovie(self, success=True): try: p = sarge.run(command_str, stderr=sarge.Capture()) if p.returncode == 0: - eventManager().fire(Events.MOVIE_DONE, {"gcode": self._gcodeFile, "movie": output, "movie_basename": os.path.basename(output)}) + eventManager().fire(Events.MOVIE_DONE, {"gcode": self._gcode_file, "movie": output, "movie_basename": os.path.basename(output)}) else: returncode = p.returncode stderr_text = p.stderr.text self._logger.warn("Could not render movie, got return code %r: %s" % (returncode, stderr_text)) - eventManager().fire(Events.MOVIE_FAILED, {"gcode": self._gcodeFile, "movie": output, "movie_basename": os.path.basename(output), "returncode": returncode, "error": stderr_text}) + eventManager().fire(Events.MOVIE_FAILED, {"gcode": self._gcode_file, "movie": output, "movie_basename": os.path.basename(output), "returncode": returncode, "error": stderr_text}) except: self._logger.exception("Could not render movie due to unknown error") - eventManager().fire(Events.MOVIE_FAILED, {"gcode": self._gcodeFile, "movie": output, "movie_basename": os.path.basename(output), "returncode": 255, "error": "Unknown error"}) + eventManager().fire(Events.MOVIE_FAILED, {"gcode": self._gcode_file, "movie": output, "movie_basename": os.path.basename(output), "returncode": 255, "error": "Unknown error"}) - def cleanCaptureDir(self): - if not os.path.isdir(self._captureDir): + def clean_capture_dir(self): + if not os.path.isdir(self._capture_dir): self._logger.warn("Cannot clean capture directory, it is unset") return - for filename in os.listdir(self._captureDir): + for filename in os.listdir(self._capture_dir): if not fnmatch.fnmatch(filename, "*.jpg"): continue - os.remove(os.path.join(self._captureDir, filename)) + os.remove(os.path.join(self._capture_dir, filename)) class ZTimelapse(Timelapse): - def __init__(self, postRoll=0, fps=25): - Timelapse.__init__(self, postRoll=postRoll, fps=fps) + def __init__(self, post_roll=0, fps=25): + Timelapse.__init__(self, post_roll=post_roll, fps=fps) self._logger.debug("ZTimelapse initialized") - def eventSubscriptions(self): + def event_subscriptions(self): return [ - (Events.Z_CHANGE, self._onZChange) + (Events.Z_CHANGE, self._on_z_change) ] - def configData(self): + def config_data(self): return { "type": "zchange" } - def processPostRoll(self): - Timelapse.processPostRoll(self) + def process_post_roll(self): + with self._capture_mutex: + filename = os.path.join(self._capture_dir, "tmp_%05d.jpg" % self._image_number) + self._image_number += 1 - filename = os.path.join(self._captureDir, "tmp_%05d.jpg" % self._imageNumber) - self._imageNumber += 1 - with self._captureMutex: - self._captureWorker(filename) + if self._perform_capture(filename): + for _ in range(self._post_roll * self._fps): + newFile = os.path.join(self._capture_dir, "tmp_%05d.jpg" % self._image_number) + self._image_number += 1 + shutil.copyfile(filename, newFile) - for i in range(self._postRoll * self._fps): - newFile = os.path.join(self._captureDir, "tmp_%05d.jpg" % (self._imageNumber)) - self._imageNumber += 1 - shutil.copyfile(filename, newFile) + Timelapse.process_post_roll(self) - if self._onPostRollDone is not None: - self._onPostRollDone() - - def _onZChange(self, event, payload): + def _on_z_change(self, event, payload): self.captureImage() class TimedTimelapse(Timelapse): - def __init__(self, postRoll=0, interval=1, fps=25): - Timelapse.__init__(self, postRoll=postRoll, fps=fps) + def __init__(self, post_roll=0, interval=1, fps=25): + Timelapse.__init__(self, post_roll=post_roll, fps=fps) self._interval = interval if self._interval < 1: self._interval = 1 # force minimum interval of 1s - self._timerThread = None + self._postroll_captures = 0 + self._timer = None self._logger.debug("TimedTimelapse initialized") + @property def interval(self): return self._interval - def configData(self): + def config_data(self): return { "type": "timed", "options": { @@ -399,25 +454,36 @@ def configData(self): } } - def onPrintStarted(self, event, payload): - Timelapse.onPrintStarted(self, event, payload) - if self._timerThread is not None: + def on_print_started(self, event, payload): + Timelapse.on_print_started(self, event, payload) + if self._timer is not None: return - self._timerThread = threading.Thread(target=self._timerWorker) - self._timerThread.daemon = True - self._timerThread.start() + self._logger.debug("Starting timer for interval based timelapse") + from octoprint.util import RepeatedTimer + self._timer = RepeatedTimer(self._interval, self._timer_task, + run_first=True, condition=self._timer_active, + on_finish=self._on_timer_finished) + self._timer.start() - def onPrintDone(self, event, payload): - Timelapse.onPrintDone(self, event, payload) - self._timerThread = None + def on_print_done(self, event, payload): + self._postroll_captures = self.post_roll * self.fps + Timelapse.on_print_done(self, event, payload) - def _timerWorker(self): - self._logger.debug("Starting timer for interval based timelapse") - while self._inTimelapse or (self._postRollStart and time.time() - self._postRollStart <= self._postRoll * self._fps): - self.captureImage() - time.sleep(self._interval) + def process_post_roll(self): + pass + + def post_roll_finished(self): + Timelapse.post_roll_finished(self) + self._timer = None + + def _timer_active(self): + return self._in_timelapse or self._postroll_captures > 0 + + def _timer_task(self): + self.captureImage() + if self._postroll_captures > 0: + self._postroll_captures -= 1 - if self._postRollStart is not None and self._onPostRollDone is not None: - self._onPostRollDone() - self._postRollStart = None + def _on_timer_finished(self): + self.post_roll_finished() From abf073340f563cf8f25215e035cd1574dd2c237f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Fri, 2 Oct 2015 15:14:05 +0200 Subject: [PATCH 17/34] Fixed reporting of duration needed for capturing timelapse postroll Needs to be calculated differently for time based and z-triggered. Capture interval was not taken properly into account. (cherry picked from commit 9284ff4) --- src/octoprint/static/js/app/dataupdater.js | 18 +++++++++++++----- src/octoprint/timelapse.py | 8 +++++++- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/octoprint/static/js/app/dataupdater.js b/src/octoprint/static/js/app/dataupdater.js index 6a63cd28b..26bd5f7e5 100644 --- a/src/octoprint/static/js/app/dataupdater.js +++ b/src/octoprint/static/js/app/dataupdater.js @@ -195,15 +195,23 @@ function DataUpdater(allViewModels) { hide: false }); } else if (type == "PostRollStart") { - if (payload.postroll_duration > 60) { - format = {duration: _.sprintf(gettext("%(minutes)d min"), {minutes: payload.postroll_duration / 60})}; + var title = gettext("Capturing timelapse postroll"); + + var text; + if (!payload.postroll_duration) { + text = _.sprintf(gettext("Now capturing timelapse post roll, this will take only a moment..."), format); } else { - format = {duration: _.sprintf(gettext("%(seconds)d sec"), {seconds: payload.postroll_duration})}; + if (payload.postroll_duration > 60) { + format = {duration: _.sprintf(gettext("%(minutes)d min"), {minutes: payload.postroll_duration / 60})}; + } else { + format = {duration: _.sprintf(gettext("%(seconds)d sec"), {seconds: payload.postroll_duration})}; + } + text = _.sprintf(gettext("Now capturing timelapse post roll, this will take approximately %(duration)s..."), format); } new PNotify({ - title: gettext("Capturing timelapse postroll"), - text: _.sprintf(gettext("Now capturing timelapse post roll, this will take approximately %(duration)s..."), format) + title: title, + text: text }); } else if (type == "SlicingStarted") { gcodeUploadProgress.addClass("progress-striped").addClass("active"); diff --git a/src/octoprint/timelapse.py b/src/octoprint/timelapse.py index f4862b73a..9ce2e3fcc 100644 --- a/src/octoprint/timelapse.py +++ b/src/octoprint/timelapse.py @@ -233,7 +233,7 @@ def f(): return f if self._post_roll > 0: - eventManager().fire(Events.POSTROLL_START, dict(postroll_duration=self.post_roll * self.fps, postroll_length=self.post_roll, postroll_fps=self.fps)) + eventManager().fire(Events.POSTROLL_START, dict(postroll_duration=self.calculate_post_roll(), postroll_length=self.post_roll, postroll_fps=self.fps)) self._post_roll_start = time.time() if doCreateMovie: self._on_post_roll_done = getWaitForCaptures(resetAndCreate) @@ -247,6 +247,9 @@ def f(): else: resetImageNumber() + def calculate_post_roll(self): + return None + def process_post_roll(self): self.post_roll_finished() @@ -470,6 +473,9 @@ def on_print_done(self, event, payload): self._postroll_captures = self.post_roll * self.fps Timelapse.on_print_done(self, event, payload) + def calculate_post_roll(self): + return self.post_roll * self.fps * self.interval + def process_post_roll(self): pass From f83d5aa89f23d64927ced85756f44d52bb0d16a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Mon, 5 Oct 2015 18:07:43 +0200 Subject: [PATCH 18/34] Fix: Use atomic writes for all save processes That includes uploaded files, profiles, caching files, settings and user directories. --- src/octoprint/filemanager/util.py | 4 +++- src/octoprint/plugins/cura/__init__.py | 2 +- src/octoprint/plugins/pluginmanager/__init__.py | 2 +- src/octoprint/settings.py | 4 +++- src/octoprint/users.py | 4 +++- 5 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/octoprint/filemanager/util.py b/src/octoprint/filemanager/util.py index 105987f37..f3fe20fb3 100644 --- a/src/octoprint/filemanager/util.py +++ b/src/octoprint/filemanager/util.py @@ -7,6 +7,8 @@ import io +from octoprint.util import atomic_write + class AbstractFileWrapper(object): """ Wrapper for file representations to save to storages. @@ -85,7 +87,7 @@ def save(self, path): """ import shutil - with open(path, "wb") as dest: + with atomic_write(path, "wb") as dest: with self.stream() as source: shutil.copyfileobj(source, dest) diff --git a/src/octoprint/plugins/cura/__init__.py b/src/octoprint/plugins/cura/__init__.py index 30de88741..fd326a109 100644 --- a/src/octoprint/plugins/cura/__init__.py +++ b/src/octoprint/plugins/cura/__init__.py @@ -393,7 +393,7 @@ def _load_profile(self, path): def _save_profile(self, path, profile, allow_overwrite=True): import yaml - with open(path, "wb") as f: + with octoprint.util.atomic_write(path, "wb") as f: yaml.safe_dump(profile, f, default_flow_style=False, indent=" ", allow_unicode=True) def _convert_to_engine(self, profile_path, printer_profile, posX, posY): diff --git a/src/octoprint/plugins/pluginmanager/__init__.py b/src/octoprint/plugins/pluginmanager/__init__.py index a93b6cc27..0c40562a1 100644 --- a/src/octoprint/plugins/pluginmanager/__init__.py +++ b/src/octoprint/plugins/pluginmanager/__init__.py @@ -536,7 +536,7 @@ def _fetch_repository_from_url(self): try: import json - with open(self._repository_cache_path, "w+b") as f: + with octoprint.util.atomic_write(self._repository_cache_path, "wb") as f: json.dump(repo_data, f) except Exception as e: self._logger.exception("Error while saving repository data to {}: {}".format(self._repository_cache_path, str(e))) diff --git a/src/octoprint/settings.py b/src/octoprint/settings.py index fa3f0e5fb..733d24b8d 100644 --- a/src/octoprint/settings.py +++ b/src/octoprint/settings.py @@ -30,6 +30,8 @@ import re import uuid +from octoprint.util import atomic_write + _APPNAME = "OctoPrint" _instance = None @@ -1057,7 +1059,7 @@ def saveScript(self, script_type, name, script): path, _ = os.path.split(filename) if not os.path.exists(path): os.makedirs(path) - with open(filename, "w+") as f: + with atomic_write(filename, "wb") as f: f.write(script) def _default_basedir(applicationName): diff --git a/src/octoprint/users.py b/src/octoprint/users.py index 76bf23bd2..d4890a642 100644 --- a/src/octoprint/users.py +++ b/src/octoprint/users.py @@ -17,6 +17,8 @@ from octoprint.settings import settings +from octoprint.util import atomic_write + class UserManager(object): valid_roles = ["user", "admin"] @@ -217,7 +219,7 @@ def _save(self, force=False): "settings": user._settings } - with open(self._userfile, "wb") as f: + with atomic_write(self._userfile, "wb") as f: yaml.safe_dump(data, f, default_flow_style=False, indent=" ", allow_unicode=True) self._dirty = False self._load() From 45c92cb1f4243febc9b5996e71ed2cde7de00825 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Mon, 5 Oct 2015 18:04:05 +0200 Subject: [PATCH 19/34] Fix: Open GCODE files als utf-8, replacing encoding errors Also detect files that contain a BOM and strip it. Internal handling of GCODE file contents switched to unicode. Should take care of #1077 --- src/octoprint/util/__init__.py | 23 +++++++++++++++++++++++ src/octoprint/util/comm.py | 13 +++++++------ src/octoprint/util/gcodeInterpreter.py | 4 +++- 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/src/octoprint/util/__init__.py b/src/octoprint/util/__init__.py index 07f80b813..58c750a38 100644 --- a/src/octoprint/util/__init__.py +++ b/src/octoprint/util/__init__.py @@ -505,6 +505,29 @@ def atomic_write(filename, mode="w+b", prefix="tmp", suffix=""): shutil.move(temp_config.name, filename) +def bom_aware_open(filename, encoding="ascii", mode="r", **kwargs): + import codecs + + codec = codecs.lookup(encoding) + encoding = codec.name + + if kwargs is None: + kwargs = dict() + + potential_bom_attribute = "BOM_" + codec.name.replace("utf-", "utf").upper() + if "r" in mode and hasattr(codecs, potential_bom_attribute): + # these encodings might have a BOM, so let's see if there is one + bom = getattr(codecs, potential_bom_attribute) + + with open(filename, "rb") as f: + header = f.read(4) + + if header.startswith(bom): + encoding += "-sig" + + return codecs.open(filename, encoding=encoding, **kwargs) + + class RepeatedTimer(threading.Thread): """ This class represents an action that should be run repeatedly in an interval. It is similar to python's diff --git a/src/octoprint/util/comm.py b/src/octoprint/util/comm.py index 49c8bc6c0..9bc2c7024 100644 --- a/src/octoprint/util/comm.py +++ b/src/octoprint/util/comm.py @@ -24,7 +24,7 @@ from octoprint.events import eventManager, Events from octoprint.filemanager import valid_file_type from octoprint.filemanager.destinations import FileDestinations -from octoprint.util import get_exception_string, sanitize_ascii, filter_non_ascii, CountedEvent, RepeatedTimer +from octoprint.util import get_exception_string, sanitize_ascii, filter_non_ascii, CountedEvent, RepeatedTimer, to_unicode, bom_aware_open try: import _winreg @@ -491,7 +491,7 @@ def fakeOk(self): self._clear_to_send.set() def sendCommand(self, cmd, cmd_type=None, processed=False): - cmd = cmd.encode('ascii', 'replace') + cmd = to_unicode(cmd, errors="replace") if not processed: cmd = process_gcode_line(cmd) if not cmd: @@ -1549,10 +1549,11 @@ def _send_loop(self): continue # now comes the part where we increase line numbers and send stuff - no turning back now + command_to_send = command.encode("ascii", errors="replace") if (gcode is not None or self._sendChecksumWithUnknownCommands) and (self.isPrinting() or self._alwaysSendChecksum): - self._doIncrementAndSendWithChecksum(command) + self._doIncrementAndSendWithChecksum(command_to_send) else: - self._doSendWithoutChecksum(command) + self._doSendWithoutChecksum(command_to_send) # trigger "sent" phase and use up one "ok" self._process_command_phase("sent", command, command_type, gcode=gcode) @@ -1952,7 +1953,7 @@ def start(self): Opens the file for reading and determines the file size. """ PrintingFileInformation.start(self) - self._handle = open(self._filename, "r") + self._handle = bom_aware_open(self._filename, encoding="utf-8", errors="replace") def close(self): """ @@ -1982,7 +1983,7 @@ def getNext(self): if self._handle is None: # file got closed just now return None - line = self._handle.readline() + line = to_unicode(self._handle.readline()) if not line: self.close() processed = process_gcode_line(line, offsets=offsets, current_tool=current_tool) diff --git a/src/octoprint/util/gcodeInterpreter.py b/src/octoprint/util/gcodeInterpreter.py index b0b69b665..853cbfd62 100644 --- a/src/octoprint/util/gcodeInterpreter.py +++ b/src/octoprint/util/gcodeInterpreter.py @@ -35,7 +35,9 @@ def load(self, filename, printer_profile, throttle=None): if os.path.isfile(filename): self.filename = filename self._fileSize = os.stat(filename).st_size - with open(filename, "r") as f: + + import codecs + with codecs.open(filename, encoding="utf-8", errors="replace") as f: self._load(f, printer_profile, throttle=throttle) def abort(self): From 548f976d35fd9831420bb884b9dee04ec28d7047 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Tue, 6 Oct 2015 13:41:54 +0200 Subject: [PATCH 20/34] Some more defensive escaping for various settings in the UI Entering HTML fragments into the webcam stream URL could cause issues, anything injected via Jinja should now be escaped properly. --- src/octoprint/server/views.py | 9 +++++++++ src/octoprint/templates/dialogs/settings.jinja2 | 2 +- .../templates/dialogs/usersettings.jinja2 | 2 +- src/octoprint/templates/index.jinja2 | 6 +++--- src/octoprint/templates/initscript.jinja2 | 16 ++++++++-------- 5 files changed, 22 insertions(+), 13 deletions(-) diff --git a/src/octoprint/server/views.py b/src/octoprint/server/views.py index 905e14366..f89bd9dad 100644 --- a/src/octoprint/server/views.py +++ b/src/octoprint/server/views.py @@ -16,11 +16,16 @@ debug, LOCALES, VERSION, DISPLAY_VERSION, UI_API_KEY, BRANCH from octoprint.settings import settings +import re + from . import util import logging _logger = logging.getLogger(__name__) +_valid_id_re = re.compile("[a-z_]+") +_valid_div_re = re.compile("[a-zA-Z_-]+") + @app.route("/") @util.flask.cached(refreshif=lambda: util.flask.cache_check_headers() or "_refresh" in request.values, key=lambda: "view/%s/%s" % (request.path, g.locale), @@ -367,6 +372,9 @@ def _process_template_config(name, implementation, rule, config=None, counter=1) data["_div"] = rule["div"](name) if "suffix" in data: data["_div"] = data["_div"] + data["suffix"] + if not _valid_div_re.match(data["_div"]): + _logger.warn("Template config {} contains invalid div identifier {}, skipping it".format(name, data["_div"])) + return None if not "template" in data: data["template"] = rule["template"](name) @@ -378,6 +386,7 @@ def _process_template_config(name, implementation, rule, config=None, counter=1) data_bind = "allowBindings: true" if "data_bind" in data: data_bind = data_bind + ", " + data["data_bind"] + data_bind = data_bind.replace("\"", "\\\"") data["data_bind"] = data_bind data["_key"] = "plugin_" + name diff --git a/src/octoprint/templates/dialogs/settings.jinja2 b/src/octoprint/templates/dialogs/settings.jinja2 index f3d3f708c..a87fa6a6a 100644 --- a/src/octoprint/templates/dialogs/settings.jinja2 +++ b/src/octoprint/templates/dialogs/settings.jinja2 @@ -20,7 +20,7 @@ class="{% if mark_active %}active{% set mark_active = False %}{% endif %} {% if "classes_link" in data %}{{ data.classes_link|join(' ') }}{% elif "classes" in data %}{{ data.classes|join(' ') }}{% endif %}" {% if "styles_link" in data %} style="{{ data.styles_link|join(', ') }}" {% elif "styles" in data %} style="{{ data.styles|join(', ') }}" {% endif %} > - {{ entry }} + {{ entry|e }} {% if "custom_bindings" not in data or data["custom_bindings"] %}{% endif %} {% endif %} diff --git a/src/octoprint/templates/dialogs/usersettings.jinja2 b/src/octoprint/templates/dialogs/usersettings.jinja2 index 00cbcc6c2..be405c153 100644 --- a/src/octoprint/templates/dialogs/usersettings.jinja2 +++ b/src/octoprint/templates/dialogs/usersettings.jinja2 @@ -17,7 +17,7 @@ class="{% if mark_active %}active{% set mark_active = False %}{% endif %} {% if "classes_link" in data %}{{ data.classes_link|join(' ') }}{% elif "classes" in data %}{{ data.classes|join(' ') }}{% endif %}" {% if "styles_link" in data %} style="{{ data.styles_link|join(', ') }}" {% elif "styles" in data %} style="{{ data.styles|join(', ') }}" {% endif %} > - {{ entry }} + {{ entry|e }} {% if "custom_bindings" not in data or data["custom_bindings"] %}{% endif %} {% endif %} diff --git a/src/octoprint/templates/index.jinja2 b/src/octoprint/templates/index.jinja2 index 1a58a82bf..a5de49c1f 100644 --- a/src/octoprint/templates/index.jinja2 +++ b/src/octoprint/templates/index.jinja2 @@ -55,7 +55,7 @@ >
- {% if "icon" in data %} {% endif %}{{ entry }} + {% if "icon" in data %} {% endif %}{{ entry|e }} {% if "template_header" in data %} {% include data.template_header ignore missing %} @@ -87,7 +87,7 @@ {% if "data_bind" in data %}data-bind="{{ data.data_bind }}"{% endif %} {% if "styles_link" in data %} style="{{ data.styles_link|join(', ') }}" {% elif "styles" in data %} style="{{ data.styles|join(', ') }}" {% endif %} > - {{ entry }} + {{ entry|e }} {% if "custom_bindings" not in data or data["custom_bindings"] %}{% endif %} {% endfor %} @@ -112,7 +112,7 @@
    -
  • {{ _('Version') }}: {{ display_version }}
    • +
  • {{ _('Version') }}: {{ display_version|e }}
  • {{ _('Homepage') }}
    • diff --git a/src/octoprint/templates/initscript.jinja2 b/src/octoprint/templates/initscript.jinja2 index c5a39dc14..8d212c621 100644 --- a/src/octoprint/templates/initscript.jinja2 +++ b/src/octoprint/templates/initscript.jinja2 @@ -10,24 +10,24 @@ var CONFIG_TIMELAPSEFILESPERPAGE = 10; var CONFIG_LOGFILESPERPAGE = 10; var CONFIG_USERSPERPAGE = 10; - var CONFIG_WEBCAM_STREAM = "{{ webcamStream }}"; + var CONFIG_WEBCAM_STREAM = "{{ webcamStream|e }}"; var CONFIG_ACCESS_CONTROL = {% if enableAccessControl -%} true; {% else %} false; {%- endif %} var CONFIG_SD_SUPPORT = {% if enableSdSupport -%} true; {% else %} false; {%- endif %} var CONFIG_FIRST_RUN = {% if firstRun -%} true; {% else %} false; {%- endif %} var CONFIG_TEMPERATURE_GRAPH = {% if enableTemperatureGraph -%} true; {% else %} false; {%- endif %} - var CONFIG_GCODE_SIZE_THRESHOLD = {{ gcodeThreshold }}; - var CONFIG_GCODE_MOBILE_SIZE_THRESHOLD = {{ gcodeMobileThreshold }}; + var CONFIG_GCODE_SIZE_THRESHOLD = {{ gcodeThreshold|e }}; + var CONFIG_GCODE_MOBILE_SIZE_THRESHOLD = {{ gcodeMobileThreshold|e }}; var SOCKJS_URI = "{{ url_for('index') }}" + "sockjs"; var SOCKJS_DEBUG = CONFIG_DEBUG; // sockjs should define CLOSE_NORMAL for us, but they don't (from ws spec) var SOCKJS_CLOSE_NORMAL = 1000; - var UI_API_KEY = "{{ uiApiKey }}"; - var VERSION = "{{ version }}"; - var DISPLAY_VERSION = "{{ display_version }}"; - var BRANCH = "{{ branch }}"; - var LOCALE = "{{ g.locale }}"; + var UI_API_KEY = "{{ uiApiKey|e }}"; + var VERSION = "{{ version|e }}"; + var DISPLAY_VERSION = "{{ display_version|e }}"; + var BRANCH = "{{ branch|e }}"; + var LOCALE = "{{ g.locale|e }}"; var AVAILABLE_LOCALES = {{ locales|tojson }}; var OCTOPRINT_VIEWMODELS = []; From 637adc3095e13d07b634978d9a2d194b2af0f4ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Tue, 6 Oct 2015 18:19:53 +0200 Subject: [PATCH 21/34] Fix: LargeResponseHandler was wrongly caching ETag value ETag values returned with download responses were cached internally within the LargeResponseHandler class, with the file's path as key. That let to problems once the content changed and hence the ETag value actually becoming invalid. Since the path however stayed the same, the same ETag value for the modified file was assumed and a 304 Not Modified response was generated. This patch changes the behaviour of LargeResponseHandler to use the last modified date of the file as the ETag value and alternatively allowing an etag generator function to be provided as constructor parameter as well to use for calculating (or disabling) the ETag header dependant on the situation. --- src/octoprint/server/util/tornado.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/octoprint/server/util/tornado.py b/src/octoprint/server/util/tornado.py index ce1bdd002..14ace0dac 100644 --- a/src/octoprint/server/util/tornado.py +++ b/src/octoprint/server/util/tornado.py @@ -776,13 +776,19 @@ class LargeResponseHandler(tornado.web.StaticFileHandler): with the requested path as parameter. Should raise a ``tornado.web.HTTPError`` (e.g. an 404) if the requested path does not pass validation in which case the request will not be further processed. Defaults to ``None`` and hence no path validation being performed. + etag_generator (function): Callback to call for generating the value of the ETag response header. Will be + called with the response handler as parameter. May return ``None`` to prevent the ETag response header + from being set. If not provided the last modified time of the file in question will be used as returned + by ``get_content_version``. """ - def initialize(self, path, default_filename=None, as_attachment=False, access_validation=None, path_validation=None): + def initialize(self, path, default_filename=None, as_attachment=False, + access_validation=None, path_validation=None, etag_generator=None): tornado.web.StaticFileHandler.initialize(self, os.path.abspath(path), default_filename) self._as_attachment = as_attachment self._access_validation = access_validation self._path_validation = path_validation + self._etag_generator = etag_generator def get(self, path, include_body=True): if self._access_validation is not None: @@ -796,6 +802,12 @@ def set_extra_headers(self, path): if self._as_attachment: self.set_header("Content-Disposition", "attachment") + def compute_etag(self): + if self._etag_generator is not None: + return self._etag_generator(self) + else: + return self.get_content_version(self.absolute_path) + @classmethod def get_content_version(cls, abspath): import os From 9cf21aa03696ae89ebdeb3139f7eebe2a8941110 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Wed, 7 Oct 2015 12:25:25 +0200 Subject: [PATCH 22/34] Also set caching headers for downloads to discourage client-side caching At least if necessary (added new boolean flag to the constructor of LargeResponseHandler for that). While at it also rename UrlForwardHandler to UrlProxyHandler (it does not forward, it proxies) and make it aware of ETag and Expires headers. --- src/octoprint/server/__init__.py | 24 ++++++++++++++++++++---- src/octoprint/server/util/tornado.py | 15 +++++++++++---- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/src/octoprint/server/__init__.py b/src/octoprint/server/__init__.py index 646e573d7..a6a6aa24e 100644 --- a/src/octoprint/server/__init__.py +++ b/src/octoprint/server/__init__.py @@ -330,13 +330,29 @@ def template_disabled(name, plugin): upload_suffixes = dict(name=s.get(["server", "uploads", "nameSuffix"]), path=s.get(["server", "uploads", "pathSuffix"])) + download_handler_kwargs = dict( + as_attachment=True, + allow_client_caching=False + ) + admin_validator = dict(access_validation=util.tornado.access_validation_factory(app, loginManager, util.flask.user_validator)) + no_hidden_files_validator = dict(path_validation=util.tornado.path_validation_factory(lambda path: not os.path.basename(path).startswith("."), status_code=404)) + + def joined_dict(*dicts): + if not len(dicts): + return dict() + + joined = dict() + for d in dicts: + joined.update(d) + return joined + server_routes = self._router.urls + [ # various downloads - (r"/downloads/timelapse/([^/]*\.mpg)", util.tornado.LargeResponseHandler, dict(path=s.getBaseFolder("timelapse"), as_attachment=True)), - (r"/downloads/files/local/(.*)", util.tornado.LargeResponseHandler, dict(path=s.getBaseFolder("uploads"), as_attachment=True, path_validation=util.tornado.path_validation_factory(lambda path: not os.path.basename(path).startswith("."), status_code=404))), - (r"/downloads/logs/([^/]*)", util.tornado.LargeResponseHandler, dict(path=s.getBaseFolder("logs"), as_attachment=True, access_validation=util.tornado.access_validation_factory(app, loginManager, util.flask.admin_validator))), + (r"/downloads/timelapse/([^/]*\.mpg)", util.tornado.LargeResponseHandler, joined_dict(dict(path=s.getBaseFolder("timelapse")), download_handler_kwargs, no_hidden_files_validator)), + (r"/downloads/files/local/(.*)", util.tornado.LargeResponseHandler, joined_dict(dict(path=s.getBaseFolder("uploads")), download_handler_kwargs, no_hidden_files_validator)), + (r"/downloads/logs/([^/]*)", util.tornado.LargeResponseHandler, joined_dict(dict(path=s.getBaseFolder("logs")), download_handler_kwargs, admin_validator)), # camera snapshot - (r"/downloads/camera/current", util.tornado.UrlForwardHandler, dict(url=s.get(["webcam", "snapshot"]), as_attachment=True, access_validation=util.tornado.access_validation_factory(app, loginManager, util.flask.user_validator))), + (r"/downloads/camera/current", util.tornado.UrlProxyHandler, dict(url=s.get(["webcam", "snapshot"]), as_attachment=True, access_validation=util.tornado.access_validation_factory(app, loginManager, util.flask.user_validator))), # generated webassets (r"/static/webassets/(.*)", util.tornado.LargeResponseHandler, dict(path=os.path.join(s.getBaseFolder("generated"), "webassets"))) ] diff --git a/src/octoprint/server/util/tornado.py b/src/octoprint/server/util/tornado.py index 14ace0dac..d69f3036a 100644 --- a/src/octoprint/server/util/tornado.py +++ b/src/octoprint/server/util/tornado.py @@ -768,6 +768,8 @@ class LargeResponseHandler(tornado.web.StaticFileHandler): :class:``~tornado.web.StaticFileHandler`` as the ``default_filename`` keyword parameter). Defaults to ``None``. as_attachment (bool): Whether to serve requested files with ``Content-Disposition: attachment`` header (``True``) or not. Defaults to ``False``. + allow_client_caching (bool): Whether to allow the client to cache (by not setting any ``Cache-Control`` or + ``Expires`` headers on the response) or not. access_validation (function): Callback to call in the ``get`` method to validate access to the resource. Will be called with ``self.request`` as parameter which contains the full tornado request object. Should raise a ``tornado.web.HTTPError`` if access is not allowed in which case the request will not be further processed. @@ -782,10 +784,11 @@ class LargeResponseHandler(tornado.web.StaticFileHandler): by ``get_content_version``. """ - def initialize(self, path, default_filename=None, as_attachment=False, + def initialize(self, path, default_filename=None, as_attachment=False, allow_client_caching=True, access_validation=None, path_validation=None, etag_generator=None): tornado.web.StaticFileHandler.initialize(self, os.path.abspath(path), default_filename) self._as_attachment = as_attachment + self._allow_client_caching = allow_client_caching self._access_validation = access_validation self._path_validation = path_validation self._etag_generator = etag_generator @@ -802,6 +805,10 @@ def set_extra_headers(self, path): if self._as_attachment: self.set_header("Content-Disposition", "attachment") + if not self._allow_client_caching: + self.set_header("Cache-Control", "max-age=0, must-revalidate, private") + self.set_header("Expires", "-1") + def compute_etag(self): if self._etag_generator is not None: return self._etag_generator(self) @@ -817,7 +824,7 @@ def get_content_version(cls, abspath): ##~~ URL Forward Handler for forwarding requests to a preconfigured static URL -class UrlForwardHandler(tornado.web.RequestHandler): +class UrlProxyHandler(tornado.web.RequestHandler): """ `tornado.web.RequestHandler `_ that proxies requests to a preconfigured url and returns the response. Allows delivery of the requested content as attachment @@ -827,7 +834,7 @@ class UrlForwardHandler(tornado.web.RequestHandler): for making the request to the configured endpoint and return the body of the client response with the status code from the client response and the following headers: - * ``Date``, ``Cache-Control``, ``Server``, ``Content-Type`` and ``Location`` will be copied over. + * ``Date``, ``Cache-Control``, ``Expires``, ``ETag``, ``Server``, ``Content-Type`` and ``Location`` will be copied over. * If ``as_attachment`` is set to True, ``Content-Disposition`` will be set to ``attachment``. If ``basename`` is set including the attachement's ``filename`` attribute will be set to the base name followed by the extension guessed based on the MIME type from the ``Content-Type`` header of the response. If no extension can be guessed @@ -877,7 +884,7 @@ def handle_response(self, response): filename = None self.set_status(response.code) - for name in ("Date", "Cache-Control", "Server", "Content-Type", "Location"): + for name in ("Date", "Cache-Control", "Server", "Content-Type", "Location", "Expires", "ETag"): value = response.headers.get(name) if value: self.set_header(name, value) From 859ea0d0e486082ed04cad398b21770756ea11e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Wed, 7 Oct 2015 17:19:18 +0200 Subject: [PATCH 23/34] Added mime type guessing for downloads to Tornado --- src/octoprint/filemanager/__init__.py | 47 ++++++++++++- src/octoprint/server/__init__.py | 7 +- src/octoprint/server/util/tornado.py | 12 +++- tests/filemanager/__init__.py | 96 +++++++++++++++++++++++++++ 4 files changed, 157 insertions(+), 5 deletions(-) diff --git a/src/octoprint/filemanager/__init__.py b/src/octoprint/filemanager/__init__.py index fad1fc41c..34f31680c 100644 --- a/src/octoprint/filemanager/__init__.py +++ b/src/octoprint/filemanager/__init__.py @@ -18,6 +18,11 @@ from .storage import LocalFileStorage from .util import AbstractFileWrapper, StreamWrapper, DiskFileWrapper +from collections import namedtuple + +ContentTypeMapping = namedtuple("ContentTypeMapping", "extensions, content_type") +ContentTypeDetector = namedtuple("ContentTypeDetector", "extensions, detector") + extensions = dict( ) @@ -25,11 +30,11 @@ def full_extension_tree(): result = dict( # extensions for 3d model files model=dict( - stl=["stl"] + stl=ContentTypeMapping(["stl"], "application/sla") ), # extensions for printable machine code machinecode=dict( - gcode=["gcode", "gco", "g"] + gcode=ContentTypeMapping(["gcode", "gco", "g"], "text/plain") ) ) @@ -68,8 +73,12 @@ def get_all_extensions(subtree=None): for key, value in subtree.items(): if isinstance(value, dict): result += get_all_extensions(value) + elif isinstance(value, (ContentTypeMapping, ContentTypeDetector)): + result += value.extensions elif isinstance(value, (list, tuple)): result += value + elif isinstance(subtree, (ContentTypeMapping, ContentTypeDetector)): + result = subtree.extensions elif isinstance(subtree, (list, tuple)): result = subtree return result @@ -79,7 +88,9 @@ def get_path_for_extension(extension, subtree=None): subtree = full_extension_tree() for key, value in subtree.items(): - if isinstance(value, (list, tuple)) and extension in value: + if isinstance(value, (ContentTypeMapping, ContentTypeDetector)) and extension in value.extensions: + return [key] + elif isinstance(value, (list, tuple)) and extension in value: return [key] elif isinstance(value, dict): path = get_path_for_extension(extension, subtree=value) @@ -88,6 +99,23 @@ def get_path_for_extension(extension, subtree=None): return None +def get_content_type_mapping_for_extension(extension, subtree=None): + if not subtree: + subtree = full_extension_tree() + + for key, value in subtree.items(): + content_extension_matches = isinstance(value, (ContentTypeMapping, ContentTypeDetector)) and extension in value. extensions + list_extension_matches = isinstance(value, (list, tuple)) and extension in value + + if content_extension_matches or list_extension_matches: + return value + elif isinstance(value, dict): + result = get_content_type_mapping_for_extension(extension, subtree=value) + if result is not None: + return result + + return None + def valid_extension(extension, type=None): if not type: return extension in get_all_extensions() @@ -106,6 +134,19 @@ def get_file_type(filename): extension = extension[1:].lower() return get_path_for_extension(extension) +def get_mime_type(filename): + _, extension = os.path.splitext(filename) + extension = extension[1:].lower() + mapping = get_content_type_mapping_for_extension(extension) + if mapping: + if isinstance(mapping, ContentTypeMapping) and mapping.content_type is not None: + return mapping.content_type + elif isinstance(mapping, ContentTypeDetector) and callable(mapping.detector): + result = mapping.detector(filename) + if result is not None: + return result + return "application/octet-stream" + class NoSuchStorage(Exception): pass diff --git a/src/octoprint/server/__init__.py b/src/octoprint/server/__init__.py index a6a6aa24e..f76f9cb8c 100644 --- a/src/octoprint/server/__init__.py +++ b/src/octoprint/server/__init__.py @@ -330,10 +330,15 @@ def template_disabled(name, plugin): upload_suffixes = dict(name=s.get(["server", "uploads", "nameSuffix"]), path=s.get(["server", "uploads", "pathSuffix"])) + def mime_type_guesser(path): + from octoprint.filemanager import get_mime_type + return get_mime_type(path) + download_handler_kwargs = dict( as_attachment=True, allow_client_caching=False ) + additional_mime_types=dict(mime_type_guesser=mime_type_guesser) admin_validator = dict(access_validation=util.tornado.access_validation_factory(app, loginManager, util.flask.user_validator)) no_hidden_files_validator = dict(path_validation=util.tornado.path_validation_factory(lambda path: not os.path.basename(path).startswith("."), status_code=404)) @@ -349,7 +354,7 @@ def joined_dict(*dicts): server_routes = self._router.urls + [ # various downloads (r"/downloads/timelapse/([^/]*\.mpg)", util.tornado.LargeResponseHandler, joined_dict(dict(path=s.getBaseFolder("timelapse")), download_handler_kwargs, no_hidden_files_validator)), - (r"/downloads/files/local/(.*)", util.tornado.LargeResponseHandler, joined_dict(dict(path=s.getBaseFolder("uploads")), download_handler_kwargs, no_hidden_files_validator)), + (r"/downloads/files/local/(.*)", util.tornado.LargeResponseHandler, joined_dict(dict(path=s.getBaseFolder("uploads")), download_handler_kwargs, no_hidden_files_validator, additional_mime_types)), (r"/downloads/logs/([^/]*)", util.tornado.LargeResponseHandler, joined_dict(dict(path=s.getBaseFolder("logs")), download_handler_kwargs, admin_validator)), # camera snapshot (r"/downloads/camera/current", util.tornado.UrlProxyHandler, dict(url=s.get(["webcam", "snapshot"]), as_attachment=True, access_validation=util.tornado.access_validation_factory(app, loginManager, util.flask.user_validator))), diff --git a/src/octoprint/server/util/tornado.py b/src/octoprint/server/util/tornado.py index d69f3036a..30753f34f 100644 --- a/src/octoprint/server/util/tornado.py +++ b/src/octoprint/server/util/tornado.py @@ -785,13 +785,15 @@ class LargeResponseHandler(tornado.web.StaticFileHandler): """ def initialize(self, path, default_filename=None, as_attachment=False, allow_client_caching=True, - access_validation=None, path_validation=None, etag_generator=None): + access_validation=None, path_validation=None, etag_generator=None, + mime_type_guesser=None): tornado.web.StaticFileHandler.initialize(self, os.path.abspath(path), default_filename) self._as_attachment = as_attachment self._allow_client_caching = allow_client_caching self._access_validation = access_validation self._path_validation = path_validation self._etag_generator = etag_generator + self._mime_type_guesser = mime_type_guesser def get(self, path, include_body=True): if self._access_validation is not None: @@ -815,6 +817,14 @@ def compute_etag(self): else: return self.get_content_version(self.absolute_path) + def get_content_type(self): + if self._mime_type_guesser is not None: + type = self._mime_type_guesser(self.absolute_path) + if type is not None: + return type + + return tornado.web.StaticFileHandler.get_content_type(self) + @classmethod def get_content_version(cls, abspath): import os diff --git a/tests/filemanager/__init__.py b/tests/filemanager/__init__.py index 65f330577..107fc063e 100644 --- a/tests/filemanager/__init__.py +++ b/tests/filemanager/__init__.py @@ -9,3 +9,99 @@ __license__ = 'GNU Affero General Public License http://www.gnu.org/licenses/agpl.html' __copyright__ = "Copyright (C) 2014 The OctoPrint Project - Released under terms of the AGPLv3 License" +import unittest +import mock + +import octoprint.filemanager + +class FilemanagerUtilTest(unittest.TestCase): + + def setUp(self): + # mock plugin manager + self.plugin_manager_patcher = mock.patch("octoprint.plugin.plugin_manager") + self.plugin_manager_getter = self.plugin_manager_patcher.start() + + self.plugin_manager = mock.MagicMock() + + hook_extensions = dict( + some_plugin=lambda: dict(dict(machinecode=dict(foo=["foo", "f"]))), + other_plugin=lambda: dict(dict(model=dict(amf=["amf"]))), + mime_map=lambda: dict( + mime_map=dict( + mime_map_yes=octoprint.filemanager.ContentTypeMapping(["mime_map_yes"], "application/mime_map_yes") + ) + ), + mime_detect=lambda: dict( + dict( + machinecode=dict( + mime_detect_yes=octoprint.filemanager.ContentTypeDetector(["mime_detect_yes"], lambda x: "application/mime_detect_yes"), + mime_detect_no=octoprint.filemanager.ContentTypeDetector(["mime_detect_no"], lambda x: None) + ) + ) + ) + ) + self.plugin_manager.get_hooks.return_value = hook_extensions + + self.plugin_manager_getter.return_value = self.plugin_manager + + def tearDown(self): + self.plugin_manager_patcher.stop() + + def test_full_extension_tree(self): + full = octoprint.filemanager.full_extension_tree() + self.assertTrue("machinecode" in full) + self.assertTrue("gcode" in full["machinecode"]) + self.assertTrue(isinstance(full["machinecode"]["gcode"], octoprint.filemanager.ContentTypeMapping)) + self.assertItemsEqual(["gcode", "gco", "g"], full["machinecode"]["gcode"].extensions) + self.assertTrue("foo" in full["machinecode"]) + self.assertTrue(isinstance(full["machinecode"]["foo"], list)) + self.assertItemsEqual(["f", "foo"], full["machinecode"]["foo"]) + + self.assertTrue("model" in full) + self.assertTrue("stl" in full["model"]) + self.assertTrue(isinstance(full["model"]["stl"], octoprint.filemanager.ContentTypeMapping)) + self.assertItemsEqual(["stl"], full["model"]["stl"].extensions) + self.assertTrue("amf" in full["model"]) + self.assertTrue(isinstance(full["model"]["amf"], list)) + self.assertItemsEqual(["amf"], full["model"]["amf"]) + + def test_get_mimetype(self): + self.assertEquals(octoprint.filemanager.get_mime_type("foo.stl"), "application/sla") + self.assertEquals(octoprint.filemanager.get_mime_type("foo.gcode"), "text/plain") + self.assertEquals(octoprint.filemanager.get_mime_type("foo.unknown"), "application/octet-stream") + self.assertEquals(octoprint.filemanager.get_mime_type("foo.mime_map_yes"), "application/mime_map_yes") + self.assertEquals(octoprint.filemanager.get_mime_type("foo.mime_map_no"), "application/octet-stream") + self.assertEquals(octoprint.filemanager.get_mime_type("foo.mime_detect_yes"), "application/mime_detect_yes") + self.assertEquals(octoprint.filemanager.get_mime_type("foo.mime_detect_no"), "application/octet-stream") + + def test_valid_file_type(self): + self.assertTrue(octoprint.filemanager.valid_file_type("foo.stl", type="model")) + self.assertTrue(octoprint.filemanager.valid_file_type("foo.stl", type="stl")) + self.assertFalse(octoprint.filemanager.valid_file_type("foo.stl", type="machinecode")) + self.assertTrue(octoprint.filemanager.valid_file_type("foo.foo", type="machinecode")) + self.assertTrue(octoprint.filemanager.valid_file_type("foo.foo", type="foo")) + self.assertTrue(octoprint.filemanager.valid_file_type("foo.foo")) + self.assertTrue(octoprint.filemanager.valid_file_type("foo.mime_map_yes")) + self.assertTrue(octoprint.filemanager.valid_file_type("foo.mime_detect_yes")) + self.assertFalse(octoprint.filemanager.valid_file_type("foo.unknown")) + + def test_get_file_type(self): + self.assertEquals(["machinecode", "gcode"], octoprint.filemanager.get_file_type("foo.gcode")) + self.assertEquals(["machinecode", "gcode"], octoprint.filemanager.get_file_type("foo.gco")) + self.assertEquals(["machinecode", "foo"], octoprint.filemanager.get_file_type("foo.f")) + self.assertEquals(["model", "stl"], octoprint.filemanager.get_file_type("foo.stl")) + self.assertEquals(["model", "amf"], octoprint.filemanager.get_file_type("foo.amf")) + self.assertIsNone(octoprint.filemanager.get_file_type("foo.unknown")) + + def test_hook_failure(self): + def hook(): + raise RuntimeError("Boo!") + self.plugin_manager.get_hooks.return_value = dict(hook=hook) + + with mock.patch("octoprint.filemanager.logging") as patched_logging: + logger = mock.MagicMock() + patched_logging.getLogger.return_value = logger + + octoprint.filemanager.get_all_extensions() + + self.assertEquals(1, len(logger.mock_calls)) From 85e6ae8e8278721affae4524c9827d63de8d8986 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Wed, 7 Oct 2015 17:45:07 +0200 Subject: [PATCH 24/34] Fixed a unit test that broke by switching to atomic_write --- tests/filemanager/test_filemanager.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/filemanager/test_filemanager.py b/tests/filemanager/test_filemanager.py index 110df373f..1da5f39f3 100644 --- a/tests/filemanager/test_filemanager.py +++ b/tests/filemanager/test_filemanager.py @@ -109,13 +109,13 @@ def test_get_metadata(self): self.assertEquals(metadata, expected) self.local_storage.get_metadata.assert_called_once_with("test.file") - @mock.patch("__builtin__.open", new_callable=mock.mock_open) + @mock.patch("octoprint.filemanager.util.atomic_write") @mock.patch("io.FileIO") @mock.patch("shutil.copyfileobj") @mock.patch("os.remove") @mock.patch("tempfile.NamedTemporaryFile") @mock.patch("time.time", side_effect=[1411979916.422, 1411979932.116]) - def test_slice(self, mocked_time, mocked_tempfile, mocked_os, mocked_shutil, mocked_fileio, mocked_open): + def test_slice(self, mocked_time, mocked_tempfile, mocked_os, mocked_shutil, mocked_fileio, mocked_atomic_write): callback = mock.MagicMock() callback_args = ("one", "two", "three") @@ -187,8 +187,8 @@ def slice(slicer_name, source_path, dest_path, profile, done_cb, printer_profile self.local_storage.add_file.assert_called_once_with("dest.file", mock.ANY, printer_profile=expected_printer_profile, allow_overwrite=True, links=expected_links) # assert that the generated gcode was manipulated as required - expected_open_calls = [mock.call("prefix/dest.file", "wb")] - self.assertEquals(mocked_open.call_args_list, expected_open_calls) + expected_atomic_write_calls = [mock.call("prefix/dest.file", "wb")] + self.assertEquals(mocked_atomic_write.call_args_list, expected_atomic_write_calls) #mocked_open.return_value.write.assert_called_once_with(";Generated from source.file aabbccddeeff\r") # assert that shutil was asked to copy the concatenated multistream From dcf2e51b21c53d52cf1539881c2523e90d4be4e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Wed, 7 Oct 2015 17:50:08 +0200 Subject: [PATCH 25/34] Moved filemanager method tests to test_filemanager.py --- tests/filemanager/__init__.py | 97 --------------------------- tests/filemanager/test_filemanager.py | 92 +++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 97 deletions(-) diff --git a/tests/filemanager/__init__.py b/tests/filemanager/__init__.py index 107fc063e..92a8db729 100644 --- a/tests/filemanager/__init__.py +++ b/tests/filemanager/__init__.py @@ -8,100 +8,3 @@ __author__ = "Gina Häußge " __license__ = 'GNU Affero General Public License http://www.gnu.org/licenses/agpl.html' __copyright__ = "Copyright (C) 2014 The OctoPrint Project - Released under terms of the AGPLv3 License" - -import unittest -import mock - -import octoprint.filemanager - -class FilemanagerUtilTest(unittest.TestCase): - - def setUp(self): - # mock plugin manager - self.plugin_manager_patcher = mock.patch("octoprint.plugin.plugin_manager") - self.plugin_manager_getter = self.plugin_manager_patcher.start() - - self.plugin_manager = mock.MagicMock() - - hook_extensions = dict( - some_plugin=lambda: dict(dict(machinecode=dict(foo=["foo", "f"]))), - other_plugin=lambda: dict(dict(model=dict(amf=["amf"]))), - mime_map=lambda: dict( - mime_map=dict( - mime_map_yes=octoprint.filemanager.ContentTypeMapping(["mime_map_yes"], "application/mime_map_yes") - ) - ), - mime_detect=lambda: dict( - dict( - machinecode=dict( - mime_detect_yes=octoprint.filemanager.ContentTypeDetector(["mime_detect_yes"], lambda x: "application/mime_detect_yes"), - mime_detect_no=octoprint.filemanager.ContentTypeDetector(["mime_detect_no"], lambda x: None) - ) - ) - ) - ) - self.plugin_manager.get_hooks.return_value = hook_extensions - - self.plugin_manager_getter.return_value = self.plugin_manager - - def tearDown(self): - self.plugin_manager_patcher.stop() - - def test_full_extension_tree(self): - full = octoprint.filemanager.full_extension_tree() - self.assertTrue("machinecode" in full) - self.assertTrue("gcode" in full["machinecode"]) - self.assertTrue(isinstance(full["machinecode"]["gcode"], octoprint.filemanager.ContentTypeMapping)) - self.assertItemsEqual(["gcode", "gco", "g"], full["machinecode"]["gcode"].extensions) - self.assertTrue("foo" in full["machinecode"]) - self.assertTrue(isinstance(full["machinecode"]["foo"], list)) - self.assertItemsEqual(["f", "foo"], full["machinecode"]["foo"]) - - self.assertTrue("model" in full) - self.assertTrue("stl" in full["model"]) - self.assertTrue(isinstance(full["model"]["stl"], octoprint.filemanager.ContentTypeMapping)) - self.assertItemsEqual(["stl"], full["model"]["stl"].extensions) - self.assertTrue("amf" in full["model"]) - self.assertTrue(isinstance(full["model"]["amf"], list)) - self.assertItemsEqual(["amf"], full["model"]["amf"]) - - def test_get_mimetype(self): - self.assertEquals(octoprint.filemanager.get_mime_type("foo.stl"), "application/sla") - self.assertEquals(octoprint.filemanager.get_mime_type("foo.gcode"), "text/plain") - self.assertEquals(octoprint.filemanager.get_mime_type("foo.unknown"), "application/octet-stream") - self.assertEquals(octoprint.filemanager.get_mime_type("foo.mime_map_yes"), "application/mime_map_yes") - self.assertEquals(octoprint.filemanager.get_mime_type("foo.mime_map_no"), "application/octet-stream") - self.assertEquals(octoprint.filemanager.get_mime_type("foo.mime_detect_yes"), "application/mime_detect_yes") - self.assertEquals(octoprint.filemanager.get_mime_type("foo.mime_detect_no"), "application/octet-stream") - - def test_valid_file_type(self): - self.assertTrue(octoprint.filemanager.valid_file_type("foo.stl", type="model")) - self.assertTrue(octoprint.filemanager.valid_file_type("foo.stl", type="stl")) - self.assertFalse(octoprint.filemanager.valid_file_type("foo.stl", type="machinecode")) - self.assertTrue(octoprint.filemanager.valid_file_type("foo.foo", type="machinecode")) - self.assertTrue(octoprint.filemanager.valid_file_type("foo.foo", type="foo")) - self.assertTrue(octoprint.filemanager.valid_file_type("foo.foo")) - self.assertTrue(octoprint.filemanager.valid_file_type("foo.mime_map_yes")) - self.assertTrue(octoprint.filemanager.valid_file_type("foo.mime_detect_yes")) - self.assertFalse(octoprint.filemanager.valid_file_type("foo.unknown")) - - def test_get_file_type(self): - self.assertEquals(["machinecode", "gcode"], octoprint.filemanager.get_file_type("foo.gcode")) - self.assertEquals(["machinecode", "gcode"], octoprint.filemanager.get_file_type("foo.gco")) - self.assertEquals(["machinecode", "foo"], octoprint.filemanager.get_file_type("foo.f")) - self.assertEquals(["model", "stl"], octoprint.filemanager.get_file_type("foo.stl")) - self.assertEquals(["model", "amf"], octoprint.filemanager.get_file_type("foo.amf")) - self.assertIsNone(octoprint.filemanager.get_file_type("foo.unknown")) - - def test_hook_failure(self): - def hook(): - raise RuntimeError("Boo!") - self.plugin_manager.get_hooks.return_value = dict(hook=hook) - - with mock.patch("octoprint.filemanager.logging") as patched_logging: - logger = mock.MagicMock() - patched_logging.getLogger.return_value = logger - - octoprint.filemanager.get_all_extensions() - - self.assertEquals(1, len(logger.mock_calls)) diff --git a/tests/filemanager/test_filemanager.py b/tests/filemanager/test_filemanager.py index 110df373f..1bb512c1b 100644 --- a/tests/filemanager/test_filemanager.py +++ b/tests/filemanager/test_filemanager.py @@ -13,6 +13,98 @@ import octoprint.filemanager import octoprint.filemanager.util +class FilemanagerMethodTest(unittest.TestCase): + + def setUp(self): + # mock plugin manager + self.plugin_manager_patcher = mock.patch("octoprint.plugin.plugin_manager") + self.plugin_manager_getter = self.plugin_manager_patcher.start() + + self.plugin_manager = mock.MagicMock() + + hook_extensions = dict( + some_plugin=lambda: dict(dict(machinecode=dict(foo=["foo", "f"]))), + other_plugin=lambda: dict(dict(model=dict(amf=["amf"]))), + mime_map=lambda: dict( + mime_map=dict( + mime_map_yes=octoprint.filemanager.ContentTypeMapping(["mime_map_yes"], "application/mime_map_yes") + ) + ), + mime_detect=lambda: dict( + dict( + machinecode=dict( + mime_detect_yes=octoprint.filemanager.ContentTypeDetector(["mime_detect_yes"], lambda x: "application/mime_detect_yes"), + mime_detect_no=octoprint.filemanager.ContentTypeDetector(["mime_detect_no"], lambda x: None) + ) + ) + ) + ) + self.plugin_manager.get_hooks.return_value = hook_extensions + + self.plugin_manager_getter.return_value = self.plugin_manager + + def tearDown(self): + self.plugin_manager_patcher.stop() + + def test_full_extension_tree(self): + full = octoprint.filemanager.full_extension_tree() + self.assertTrue("machinecode" in full) + self.assertTrue("gcode" in full["machinecode"]) + self.assertTrue(isinstance(full["machinecode"]["gcode"], octoprint.filemanager.ContentTypeMapping)) + self.assertItemsEqual(["gcode", "gco", "g"], full["machinecode"]["gcode"].extensions) + self.assertTrue("foo" in full["machinecode"]) + self.assertTrue(isinstance(full["machinecode"]["foo"], list)) + self.assertItemsEqual(["f", "foo"], full["machinecode"]["foo"]) + + self.assertTrue("model" in full) + self.assertTrue("stl" in full["model"]) + self.assertTrue(isinstance(full["model"]["stl"], octoprint.filemanager.ContentTypeMapping)) + self.assertItemsEqual(["stl"], full["model"]["stl"].extensions) + self.assertTrue("amf" in full["model"]) + self.assertTrue(isinstance(full["model"]["amf"], list)) + self.assertItemsEqual(["amf"], full["model"]["amf"]) + + def test_get_mimetype(self): + self.assertEquals(octoprint.filemanager.get_mime_type("foo.stl"), "application/sla") + self.assertEquals(octoprint.filemanager.get_mime_type("foo.gcode"), "text/plain") + self.assertEquals(octoprint.filemanager.get_mime_type("foo.unknown"), "application/octet-stream") + self.assertEquals(octoprint.filemanager.get_mime_type("foo.mime_map_yes"), "application/mime_map_yes") + self.assertEquals(octoprint.filemanager.get_mime_type("foo.mime_map_no"), "application/octet-stream") + self.assertEquals(octoprint.filemanager.get_mime_type("foo.mime_detect_yes"), "application/mime_detect_yes") + self.assertEquals(octoprint.filemanager.get_mime_type("foo.mime_detect_no"), "application/octet-stream") + + def test_valid_file_type(self): + self.assertTrue(octoprint.filemanager.valid_file_type("foo.stl", type="model")) + self.assertTrue(octoprint.filemanager.valid_file_type("foo.stl", type="stl")) + self.assertFalse(octoprint.filemanager.valid_file_type("foo.stl", type="machinecode")) + self.assertTrue(octoprint.filemanager.valid_file_type("foo.foo", type="machinecode")) + self.assertTrue(octoprint.filemanager.valid_file_type("foo.foo", type="foo")) + self.assertTrue(octoprint.filemanager.valid_file_type("foo.foo")) + self.assertTrue(octoprint.filemanager.valid_file_type("foo.mime_map_yes")) + self.assertTrue(octoprint.filemanager.valid_file_type("foo.mime_detect_yes")) + self.assertFalse(octoprint.filemanager.valid_file_type("foo.unknown")) + + def test_get_file_type(self): + self.assertEquals(["machinecode", "gcode"], octoprint.filemanager.get_file_type("foo.gcode")) + self.assertEquals(["machinecode", "gcode"], octoprint.filemanager.get_file_type("foo.gco")) + self.assertEquals(["machinecode", "foo"], octoprint.filemanager.get_file_type("foo.f")) + self.assertEquals(["model", "stl"], octoprint.filemanager.get_file_type("foo.stl")) + self.assertEquals(["model", "amf"], octoprint.filemanager.get_file_type("foo.amf")) + self.assertIsNone(octoprint.filemanager.get_file_type("foo.unknown")) + + def test_hook_failure(self): + def hook(): + raise RuntimeError("Boo!") + self.plugin_manager.get_hooks.return_value = dict(hook=hook) + + with mock.patch("octoprint.filemanager.logging") as patched_logging: + logger = mock.MagicMock() + patched_logging.getLogger.return_value = logger + + octoprint.filemanager.get_all_extensions() + + self.assertEquals(1, len(logger.mock_calls)) + class FileManagerTest(unittest.TestCase): def setUp(self): From 947f4c89cc588ab7b5d9a18ca23ce7cba6d989cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Wed, 7 Oct 2015 18:13:17 +0200 Subject: [PATCH 26/34] Fixed a win-only issue causing the gcode view not to reload on file modification --- src/octoprint/printer/standard.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/octoprint/printer/standard.py b/src/octoprint/printer/standard.py index c332182c5..8e11b6421 100644 --- a/src/octoprint/printer/standard.py +++ b/src/octoprint/printer/standard.py @@ -680,7 +680,7 @@ def _setJobData(self, filename, filesize, sd): # Use a string for mtime because it could be float and the # javascript needs to exact match if not sd: - date = int(os.stat(path_on_disk).st_ctime) + date = int(os.stat(path_on_disk).st_mtime) try: fileData = self._fileManager.get_metadata(FileDestinations.SDCARD if sd else FileDestinations.LOCAL, path_on_disk) From bbbdb3dde6aa9811d5a2fe7e46539d44d2c24094 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Thu, 8 Oct 2015 10:22:09 +0200 Subject: [PATCH 27/34] Note that z-based timelapses don't work when printing from SD --- src/octoprint/templates/tabs/timelapse.jinja2 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/octoprint/templates/tabs/timelapse.jinja2 b/src/octoprint/templates/tabs/timelapse.jinja2 index 5fda40a9f..acea78ade 100644 --- a/src/octoprint/templates/tabs/timelapse.jinja2 +++ b/src/octoprint/templates/tabs/timelapse.jinja2 @@ -7,10 +7,11 @@ {{ _('Warning') }} {{ _('Do not use with spiralized ("Joris") vases or similar continuous Z models.') }} + {{ _('Note') }} {% trans %}Does not work when printing from the printer's SD Card (no way to detect the change in Z reliably). Use "Timed" mode for those prints instead.{% endtrans %}
    From c932e496fa461ac0a86d73842d728c7c3c7c0cba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Thu, 8 Oct 2015 17:08:36 +0200 Subject: [PATCH 28/34] SWU: Show reload dialog after update The notification and the (forced) dialog afterwards were way too redundant. --- .../softwareupdate/static/js/softwareupdate.js | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/src/octoprint/plugins/softwareupdate/static/js/softwareupdate.js b/src/octoprint/plugins/softwareupdate/static/js/softwareupdate.js index fd4ba14ff..8b75047b4 100644 --- a/src/octoprint/plugins/softwareupdate/static/js/softwareupdate.js +++ b/src/octoprint/plugins/softwareupdate/static/js/softwareupdate.js @@ -27,6 +27,8 @@ $(function() { {"key": "git_commit", "name": gettext("Commit")} ]; + self.reloadOverlay = $("#reloadui_overlay"); + self.versions = new ItemListHelper( "plugin.softwareupdate.versions", { @@ -329,18 +331,10 @@ $(function() { self.onDataUpdaterReconnect = function() { if (self.waitingForRestart) { self.waitingForRestart = false; - - var options = { - title: gettext("Restart successful!"), - text: gettext("The server was restarted successfully. The page will now reload automatically."), - type: "success", - hide: false - }; - self._showPopup(options); self.updateInProgress = false; - - var delay = 5 + Math.floor(Math.random() * 5) + 1; - setTimeout(function() {location.reload(true);}, delay * 1000); + if (!self.reloadOverlay.is(":visible")) { + self.reloadOverlay.show(); + } } }; From c55fea666db6bd2b95429b82fcde9f1298f47a53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Mon, 19 Oct 2015 09:33:48 +0200 Subject: [PATCH 29/34] Updated required versions of psutil and netifaces Also see #1090 --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 0fed3ef0c..80ed83c4b 100644 --- a/setup.py +++ b/setup.py @@ -26,13 +26,13 @@ "netaddr==0.7.17", "watchdog==0.8.3", "sarge==0.1.4", - "netifaces==0.8", + "netifaces==0.10", "pylru==1.0.9", "rsa==3.2", "pkginfo==1.2.1", "requests==2.7.0", "semantic_version==2.4.2", - "psutil==3.1.1" + "psutil==3.2.1" ] # Additional requirements for optional install options From 758062133cafa9593d0eabd7094a35c6583ab83b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Tue, 20 Oct 2015 11:55:34 +0200 Subject: [PATCH 30/34] Updated changelog for 1.2.7 --- CHANGELOG.md | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 541f25515..d9597bc5b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,36 @@ # OctoPrint Changelog +## 1.2.7 (2015-10-20) + +### Improvements + + * [#1062](https://github.com/foosel/OctoPrint/issues/1062) - Plugin Manager now has a configuration dialog that among other things allows defining the used `pip` command if auto detection proves to be insufficient here. + * Allow defining additional `pip` parameters in Plugin Manager. That might make `sudo`-less installation of plugins possible in situations where it's tricky otherwise (e.g. by using `--user`) + * Improved timelapse processing (backported from `devel` branch): + * Individually captured frames cannot "overtake" each other anymore through usage of a capture queue. + * Notifications will now be shown when the capturing of the timelapse's post roll happens, including an approximation of how long that will take. + * Usage of `requests` instead of `urllib` for fetching the snapshot, appears to also have [positive effects on webcam compatibility](https://github.com/foosel/OctoPrint/issues/1078). + * Some more defensive escaping for various settings in the UI (e.g. webcam URL) + * Switch to more error resilient saving of configuration files and other files modified during runtime (save to temporary file & move). Should reduce risk of file corruption. + * Downloading GCODE and STL files should now set more fitting `Content-Type` headers (`text/plain` and `application/sla`) for better client side support for "Open with" like usage scenarios. + * Selecting z-triggered timelapse mode will now inform about that not working when printing from SD card. + * Software Update Plugin: Removed "The web interface will now be reloaded" notification after successful update since that became obsolete with introduction of the "Reload Now" overlay. + * Updated required version of `psutil` and `netifaces` dependencies. + +### Bug Fixes + + * [#1057](https://github.com/foosel/OctoPrint/issues/1057) - Better error resilience of the Software Update plugin against broken/incomplete update configurations. + * [#1075](https://github.com/foosel/OctoPrint/issues/1075) - Fixed support of `sudo` for installing plugins, but added big visible warning about it as it's **not** recommended. + * [#1077](https://github.com/foosel/OctoPrint/issues/1077) - Do not hiccup on [UTF-8 BOMs](https://en.wikipedia.org/wiki/Byte_order_mark) (or other BOMs for that matter) at the beginning of GCODE files. + * Fixed an issue that caused user sessions to not be properly associated, leading to Sessions getting duplicated, wrongly saved etc. + * Fixed internal server error (HTTP 500) response on REST API calls with unset `Content-Type` header. + * Fixed an issue leading to drag-and-drop file uploads to trigger frontend processing in various other file upload widgets. + * Fixed a documentation error. + * Fixed caching behaviour on GCODE/STL downloads, was setting the `ETag` header improperly. + * Fixed GCODE viewer not properly detecting change of currently visualized file on Windows systems. + +([Commits](https://github.com/foosel/OctoPrint/compare/1.2.6...1.2.7)) + ## 1.2.6 (2015-09-02) ### Improvements From 693633bd1d51baf2a57bb628757c326ed540e359 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Mon, 20 Jul 2015 16:29:29 +0200 Subject: [PATCH 31/34] RepeatedTimer now supports callback when timer stops (cherry picked from commit 3c5a976) --- src/octoprint/util/__init__.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/octoprint/util/__init__.py b/src/octoprint/util/__init__.py index 58c750a38..b09dd252f 100644 --- a/src/octoprint/util/__init__.py +++ b/src/octoprint/util/__init__.py @@ -585,10 +585,13 @@ def hello(): run_first (boolean): If set to True, the function will be run for the first time *before* the first wait period. If set to False (the default), the function will be run for the first time *after* the first wait period. condition (callable): Condition that needs to be True for loop to continue. Defaults to ``lambda: True``. + on_finish (callable): Callback to call when the timer finishes, either due to being cancelled or since + the condition became false. daemon (bool): daemon flag to set on underlying thread. """ - def __init__(self, interval, function, args=None, kwargs=None, run_first=False, condition=None, daemon=True): + def __init__(self, interval, function, args=None, kwargs=None, + run_first=False, condition=None, on_finish=None, daemon=True): threading.Thread.__init__(self) if args is None: @@ -609,10 +612,11 @@ def __init__(self, interval, function, args=None, kwargs=None, run_first=False, self.kwargs = kwargs self.run_first = run_first self.condition = condition + self.on_finish = on_finish self.daemon = daemon def cancel(self): - self.finished.set() + self.finish() def run(self): while self.condition(): @@ -633,8 +637,13 @@ def run(self): # if we are to run the function AFTER waiting for the first time self.function(*self.args, **self.kwargs) + self.finish() + + def finish(self): # make sure we set our finished event so we can detect that the loop was finished self.finished.set() + if callable(self.on_finish): + self.on_finish() class CountedEvent(object): From aa5099340ea9aaab6550c7a39f7928b8262880a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Fri, 28 Aug 2015 16:53:22 +0200 Subject: [PATCH 32/34] Two new callbacks for RepeatedTimer when cancelled or condition untrue (cherry picked from commit 9b9ecfe) --- src/octoprint/util/__init__.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/src/octoprint/util/__init__.py b/src/octoprint/util/__init__.py index b09dd252f..bc4ddfd71 100644 --- a/src/octoprint/util/__init__.py +++ b/src/octoprint/util/__init__.py @@ -585,13 +585,18 @@ def hello(): run_first (boolean): If set to True, the function will be run for the first time *before* the first wait period. If set to False (the default), the function will be run for the first time *after* the first wait period. condition (callable): Condition that needs to be True for loop to continue. Defaults to ``lambda: True``. + on_condition_false (callable): Callback to call when the timer finishes due to condition becoming false. Will + be called before the ``on_finish`` callback. + on_cancelled (callable): Callback to call when the timer finishes due to being cancelled. Will be called + before the ``on_finish`` callback. on_finish (callable): Callback to call when the timer finishes, either due to being cancelled or since the condition became false. daemon (bool): daemon flag to set on underlying thread. """ def __init__(self, interval, function, args=None, kwargs=None, - run_first=False, condition=None, on_finish=None, daemon=True): + run_first=False, condition=None, on_condition_false=None, + on_cancelled=None, on_finish=None, daemon=True): threading.Thread.__init__(self) if args is None: @@ -612,11 +617,15 @@ def __init__(self, interval, function, args=None, kwargs=None, self.kwargs = kwargs self.run_first = run_first self.condition = condition + + self.on_condition_false = on_condition_false + self.on_cancelled = on_cancelled self.on_finish = on_finish + self.daemon = daemon def cancel(self): - self.finish() + self._finish(self.on_cancelled) def run(self): while self.condition(): @@ -631,17 +640,23 @@ def run(self): # wait, but break if we are cancelled self.finished.wait(self.interval()) if self.finished.is_set(): - break + return if not self.run_first: # if we are to run the function AFTER waiting for the first time self.function(*self.args, **self.kwargs) - self.finish() + # we'll only get here if the condition was false + self._finish(self.on_condition_false) - def finish(self): - # make sure we set our finished event so we can detect that the loop was finished + def _finish(self, *callbacks): self.finished.set() + + for callback in callbacks: + if not callable(callback): + continue + callback() + if callable(self.on_finish): self.on_finish() From d34790e74e1811f926002e96c181070f1b3cdd6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Fri, 28 Aug 2015 16:56:14 +0200 Subject: [PATCH 33/34] Unit tests for octoprint.util.RepeatedTimer (cherry picked from commit 7362067) --- tests/util/test_repeated_timer.py | 180 ++++++++++++++++++++++++++++++ 1 file changed, 180 insertions(+) create mode 100644 tests/util/test_repeated_timer.py diff --git a/tests/util/test_repeated_timer.py b/tests/util/test_repeated_timer.py new file mode 100644 index 000000000..6eed773b9 --- /dev/null +++ b/tests/util/test_repeated_timer.py @@ -0,0 +1,180 @@ +# coding=utf-8 +from __future__ import absolute_import + +__author__ = "Gina Häußge " +__license__ = 'GNU Affero General Public License http://www.gnu.org/licenses/agpl.html' +__copyright__ = "Copyright (C) 2015 The OctoPrint Project - Released under terms of the AGPLv3 License" + +import unittest +import mock +import time + +from octoprint.util import RepeatedTimer + + +class Countdown(object): + def __init__(self, start): + self._counter = start + + def step(self): + self._counter -= 1 + + @property + def counter(self): + return self._counter + + +class IncreasingInterval(Countdown): + def __init__(self, start, factor): + Countdown.__init__(self, start) + self._start = start + self._factor = factor + + def interval(self): + result = (self._start - self._counter + 1) * self._factor + return result + + +class RepeatedTimerTest(unittest.TestCase): + + def setUp(self): + pass + + def test_condition(self): + countdown = Countdown(5) + timer_task = mock.MagicMock() + timer_task.side_effect = countdown.step + + timer = RepeatedTimer(0.1, timer_task, condition=lambda: countdown.counter > 0) + timer.start() + + # wait for it + timer.join() + + self.assertEquals(5, timer_task.call_count) + + def test_finished_callback(self): + countdown = Countdown(5) + timer_task = mock.MagicMock() + timer_task.side_effect = countdown.step + + on_finished = mock.MagicMock() + + timer = RepeatedTimer(0.1, timer_task, condition=lambda: countdown.counter > 0, on_finish=on_finished) + timer.start() + + # wait for it + timer.join() + + self.assertEquals(1, on_finished.call_count) + + def test_condition_callback(self): + countdown = Countdown(5) + timer_task = mock.MagicMock() + timer_task.side_effect = countdown.step + + on_cancelled = mock.MagicMock() + on_condition_false = mock.MagicMock() + + timer = RepeatedTimer(0.1, timer_task, + condition=lambda: countdown.counter > 0, + on_condition_false=on_condition_false, + on_cancelled=on_cancelled) + timer.start() + + # wait for it + timer.join() + + self.assertEquals(1, on_condition_false.call_count) + self.assertEquals(0, on_cancelled.call_count) + + def test_cancelled_callback(self): + countdown = Countdown(5) + timer_task = mock.MagicMock() + timer_task.side_effect = countdown.step + + on_cancelled = mock.MagicMock() + on_condition_false = mock.MagicMock() + + timer = RepeatedTimer(10, timer_task, + condition=lambda: countdown.counter > 0, + on_condition_false=on_condition_false, + on_cancelled=on_cancelled) + timer.start() + + # give it some time to run + time.sleep(1) + + # then cancel it and wait for the thread to really finish + timer.cancel() + timer.join() + + self.assertEquals(0, on_condition_false.call_count) + self.assertEquals(1, on_cancelled.call_count) + + def test_run_first(self): + timer_task = mock.MagicMock() + + timer = RepeatedTimer(60, timer_task, run_first=True) + timer.start() + + # give it some time to run + time.sleep(1) + + # then cancel it and wait for the thread to really finish + timer.cancel() + timer.join() + + # should have run once + self.assertEquals(1, timer_task.call_count) + + def test_not_run_first(self): + timer_task = mock.MagicMock() + + timer = RepeatedTimer(60, timer_task) + timer.start() + + # give it some time to run - should hang in the sleep phase though + time.sleep(1) + + # then cancel it and wait for the thread to really finish + timer.cancel() + timer.join() + + self.assertEquals(0, timer_task.call_count) + + def test_adjusted_interval(self): + increasing_interval = IncreasingInterval(3, 1) + + timer_task = mock.MagicMock() + timer_task.side_effect = increasing_interval.step + + timer = RepeatedTimer(increasing_interval.interval, + timer_task, + condition=lambda: increasing_interval.counter > 0) + + # this should take 1 + 2 + 3 = 6s + start_time = time.time() + timer.start() + timer.join() + duration = time.time() - start_time + + self.assertEquals(3, timer_task.call_count) + self.assertGreaterEqual(duration, 6) + self.assertLess(duration, 7) + + def test_condition_change_during_task(self): + def sleep(): + time.sleep(2) + + timer_task = mock.MagicMock() + timer_task.side_effect = sleep + + timer = RepeatedTimer(0.1, timer_task, run_first=True) + timer.start() + + time.sleep(1) + timer.condition = lambda: False + timer.join() + + self.assertEquals(1, timer_task.call_count) From 6c622f7c4332b71c6ece59552ffc87c146155c84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Tue, 20 Oct 2015 12:24:23 +0200 Subject: [PATCH 34/34] Fixed indentation of change log --- CHANGELOG.md | 59 +++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 42 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d9597bc5b..be40008fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,30 +4,55 @@ ### Improvements - * [#1062](https://github.com/foosel/OctoPrint/issues/1062) - Plugin Manager now has a configuration dialog that among other things allows defining the used `pip` command if auto detection proves to be insufficient here. - * Allow defining additional `pip` parameters in Plugin Manager. That might make `sudo`-less installation of plugins possible in situations where it's tricky otherwise (e.g. by using `--user`) + * [#1062](https://github.com/foosel/OctoPrint/issues/1062) - Plugin Manager + now has a configuration dialog that among other things allows defining the + used `pip` command if auto detection proves to be insufficient here. + * Allow defining additional `pip` parameters in Plugin Manager. That might + make `sudo`-less installation of plugins possible in situations where it's + tricky otherwise. * Improved timelapse processing (backported from `devel` branch): - * Individually captured frames cannot "overtake" each other anymore through usage of a capture queue. - * Notifications will now be shown when the capturing of the timelapse's post roll happens, including an approximation of how long that will take. - * Usage of `requests` instead of `urllib` for fetching the snapshot, appears to also have [positive effects on webcam compatibility](https://github.com/foosel/OctoPrint/issues/1078). + * Individually captured frames cannot "overtake" each other anymore through + usage of a capture queue. + * Notifications will now be shown when the capturing of the timelapse's + post roll happens, including an approximation of how long that will take. + * Usage of `requests` instead of `urllib` for fetching the snapshot, + appears to also have [positive effects on webcam compatibility](https://github.com/foosel/OctoPrint/issues/1078). * Some more defensive escaping for various settings in the UI (e.g. webcam URL) - * Switch to more error resilient saving of configuration files and other files modified during runtime (save to temporary file & move). Should reduce risk of file corruption. - * Downloading GCODE and STL files should now set more fitting `Content-Type` headers (`text/plain` and `application/sla`) for better client side support for "Open with" like usage scenarios. - * Selecting z-triggered timelapse mode will now inform about that not working when printing from SD card. - * Software Update Plugin: Removed "The web interface will now be reloaded" notification after successful update since that became obsolete with introduction of the "Reload Now" overlay. + * Switch to more error resilient saving of configuration files and other files + modified during runtime (save to temporary file & move). Should reduce risk + of file corruption. + * Downloading GCODE and STL files should now set more fitting `Content-Type` + headers (`text/plain` and `application/sla`) for better client side support + for "Open with" like usage scenarios. + * Selecting z-triggered timelapse mode will now inform about that not working + when printing from SD card. + * Software Update Plugin: Removed "The web interface will now be reloaded" + notification after successful update since that became obsolete with + introduction of the "Reload Now" overlay. * Updated required version of `psutil` and `netifaces` dependencies. ### Bug Fixes - * [#1057](https://github.com/foosel/OctoPrint/issues/1057) - Better error resilience of the Software Update plugin against broken/incomplete update configurations. - * [#1075](https://github.com/foosel/OctoPrint/issues/1075) - Fixed support of `sudo` for installing plugins, but added big visible warning about it as it's **not** recommended. - * [#1077](https://github.com/foosel/OctoPrint/issues/1077) - Do not hiccup on [UTF-8 BOMs](https://en.wikipedia.org/wiki/Byte_order_mark) (or other BOMs for that matter) at the beginning of GCODE files. - * Fixed an issue that caused user sessions to not be properly associated, leading to Sessions getting duplicated, wrongly saved etc. - * Fixed internal server error (HTTP 500) response on REST API calls with unset `Content-Type` header. - * Fixed an issue leading to drag-and-drop file uploads to trigger frontend processing in various other file upload widgets. + * [#1057](https://github.com/foosel/OctoPrint/issues/1057) - Better error + resilience of the Software Update plugin against broken/incomplete update + configurations. + * [#1075](https://github.com/foosel/OctoPrint/issues/1075) - Fixed support + of `sudo` for installing plugins, but added big visible warning about it + as it's **not** recommended. + * [#1077](https://github.com/foosel/OctoPrint/issues/1077) - Do not hiccup + on [UTF-8 BOMs](https://en.wikipedia.org/wiki/Byte_order_mark) (or other + BOMs for that matter) at the beginning of GCODE files. + * Fixed an issue that caused user sessions to not be properly associated, + leading to Sessions getting duplicated, wrongly saved etc. + * Fixed internal server error (HTTP 500) response on REST API calls with + unset `Content-Type` header. + * Fixed an issue leading to drag-and-drop file uploads to trigger frontend + processing in various other file upload widgets. * Fixed a documentation error. - * Fixed caching behaviour on GCODE/STL downloads, was setting the `ETag` header improperly. - * Fixed GCODE viewer not properly detecting change of currently visualized file on Windows systems. + * Fixed caching behaviour on GCODE/STL downloads, was setting the `ETag` + header improperly. + * Fixed GCODE viewer not properly detecting change of currently visualized + file on Windows systems. ([Commits](https://github.com/foosel/OctoPrint/compare/1.2.6...1.2.7))