2.4.0 (#201)

* Suppress core OctoPrint GCODE scripts when running CPQ scripts of our own

* Also protect the patchComms() function from exceptions e.g. due to OctoPrint refactors

* Also return original script values in passthru

* Configurable GCODE override in print files

* Properly order sets by rank

* Preprocessor simulator and preview (#194)

* Partial commit - extended CustomEvents to include sim stuff, added sim api endpoint, split out settings JS into a separate _event.js file

* Fixed simulation execution events, added styling, plus modifiable symtable and display of mutations

* Pluralize simulation summary, separate asteval code into automation.py from storage/queries.py, set up tests

* Fix tests

* Add tests for CPSettingsEvent

* Make simulator UI more friendly/helpful and well formatted

* Remove todo lines

* Tighten spoolmanager integration (#195)

* Tighten spoolmanager integration, including validation checks and print-start adjustments

* Add tests, improve set_status behavior, add spoolmanager.py files

* Fix tests, eliminate stack overflow

* Increase idle timeout seconds

* Add preprocessor basics to docs (#196)

* Add S3D processor (#197)

* Compute stats for whole queues (#198)

* Extract stats calculations from continuousprint_jobs.js, make also usable on whole queues

* Add HTML for viewing queue rollups, make selectively visible

* Fix tests, fix batch selection, auto-hide mass estimates when zero

* Auto-slice STLs in queue (#166)

* First attempt at integrating OctoPrint configured slicers into queue automation; also anticipate FILE_ADDED changes upstream

* Added auto-slicing docs and fixed small bug in octoprint version detection

* Add working auto-slicer implementation; unit tests TBD

* Fix tests

* Add driver test

* Add tests for slicer profiles in JS, remove extra debug logging

* Remove unused driver state, improve verbosity of slicing errors, and fixed slicer output path in tmp folder

* Get working enough to verify STL and gcode handled correctly, plus update docs

* Fix tests, remove octoprint version req thing and reduce DB noise in testing

* Cleanup

* 2.4.0rc1

* Creality Ender 3 S1 Pro added (#207)

* Ender 3 S1 Pro bed cleaning (#208)

* Update gcode_scripts.yaml

* Update gcode_scripts.yaml

* Add safety temp directory creation, fixed YAML data files

* Appease linter

* Fix tests and broken links

---------

Co-authored-by: UdDReX <44963788+UdDReX@users.noreply.github.com>

Dockerfile

@@ -1,7 +1,8 @@
 FROM python:3.7
 
 # Installing ffmpeg is needed for working with timelapses - can be ommitted otherwise
-RUN apt-get update && apt-get -y install --no-install-recommends ffmpeg && rm -rf /var/lib/apt/lists/*
+# Also install vim for later edit based debugging
+RUN apt-get update && apt-get -y install --no-install-recommends ffmpeg vim && rm -rf /var/lib/apt/lists/*
 
 # IPFS installation for LAN filesharing
 RUN wget https://dist.ipfs.tech/kubo/v0.15.0/kubo_v0.15.0_linux-amd64.tar.gz \

continuousprint/__init__.py

@@ -12,6 +12,7 @@
     ASSETS,
     TEMPLATES,
     update_info,
+    SIMULATOR_DEFAULT_SYMTABLE,
 )
 from .storage import queries
 from .api import Permission as CPQPermission
@@ -26,7 +27,6 @@ class ContinuousprintPlugin(
     octoprint.plugin.StartupPlugin,
     octoprint.plugin.EventHandlerPlugin,
 ):
-
     # -------------------- Begin BlueprintPlugin --------------------
 
     def get_blueprint(self):
@@ -53,6 +53,7 @@ def on_startup(self, host=None, port=None):
                 self._printer,
                 self._settings,
                 self._file_manager,
+                self._slicing_manager,
                 self._plugin_manager,
                 queries,
                 self.get_plugin_data_folder(),
@@ -64,6 +65,13 @@ def on_startup(self, host=None, port=None):
             )
 
     def on_after_startup(self):
+        self._logger.debug(
+            "Starting ContinuousPrint with settings: {}".format(
+                self._settings.get_all_data()
+            )
+        )
+        self._plugin.patchCommJobReader()
+        self._plugin.patchComms()
         self._plugin.start()
 
         # It's possible to miss events or for some weirdness to occur in conditionals. Adding a watchdog
@@ -108,6 +116,7 @@ def get_template_vars(self):
             gcode_scripts=list(GCODE_SCRIPTS.values()),
             custom_events=[e.as_dict() for e in CustomEvents],
             local_ip=local_ip,
+            simulator_default_symtable=SIMULATOR_DEFAULT_SYMTABLE,
         )
 
     def get_template_configs(self):

continuousprint/api.py

@@ -3,10 +3,12 @@
 from octoprint.access.permissions import Permissions, ADMIN_GROUP
 from octoprint.server.util.flask import restricted_access
 from .queues.lan import ValidationError
+from .automation import getInterpreter, genEventScript
 import flask
 import json
 from .storage import queries
 from .storage.database import DEFAULT_QUEUE
+from .data import CustomEvents
 from .driver import Action as DA
 from abc import ABC, abstractmethod
 
@@ -344,3 +346,27 @@ def get_automation(self):
     def set_automation_external_symbols(self):
         self._set_external_symbols(flask.request.get_json())
         return json.dumps("OK")
+
+    # PRIVATE API METHOD - may change without warning.
+    @octoprint.plugin.BlueprintPlugin.route("/automation/simulate", methods=["POST"])
+    @restricted_access
+    @cpq_permission(Permission.EDITAUTOMATION)
+    def simulate_automation(self):
+        symtable = json.loads(flask.request.form.get("symtable"))
+        automation = json.loads(flask.request.form.get("automation"))
+        interp, out, err = getInterpreter(symtable)
+        symtable = interp.symtable.copy()  # Pick up defaults
+        result = dict(
+            gcode=genEventScript(automation, interp),
+            symtable_diff={},
+        )
+
+        err.seek(0)
+        result["stderr"] = err.read()
+        out.seek(0)
+        result["stdout"] = out.read()
+        for k, v in interp.symtable.items():
+            if k not in symtable or symtable[k] != v:
+                result["symtable_diff"][k] = repr(v)
+        self._logger.debug(f"Simulator result: {result}")
+        return json.dumps(result)

continuousprint/api_test.py

@@ -2,7 +2,7 @@
 import json
 import logging
 from .driver import Action as DA
-from unittest.mock import patch, MagicMock, call
+from unittest.mock import patch, MagicMock, call, PropertyMock
 import imp
 from flask import Flask
 from .api import Permission, cpq_permission
@@ -87,6 +87,7 @@ def test_role_access_denied(self):
             ("GETAUTOMATION", "/automation/get"),
             ("EDITAUTOMATION", "/automation/edit"),
             ("EDITAUTOMATION", "/automation/external"),
+            ("EDITAUTOMATION", "/automation/simulate"),
         ]
         self.api._get_queue = None  # MagicMock interferes with checking
 
@@ -102,7 +103,7 @@ def test_role_access_denied(self):
         num_perms = len([p for p in Permission])
         self.assertEqual(num_perms_tested, num_perms)
 
-        for (role, endpoint) in testcases:
+        for role, endpoint in testcases:
             p = getattr(self.perm, f"PLUGIN_CONTINUOUSPRINT_{role}")
             p.can.return_value = False
             if role.startswith("GET"):
@@ -301,3 +302,36 @@ def test_automation_external(self, q):
         rep = self.client.post("/automation/external", json=dict(foo="bar"))
         self.assertEqual(rep.status_code, 200)
         self.api._set_external_symbols.assert_called_with(dict(foo="bar"))
+
+    @patch("continuousprint.api.getInterpreter")
+    @patch("continuousprint.api.genEventScript")
+    def test_automation_simulate(self, ge, gi):
+        self.perm.PLUGIN_CONTINUOUSPRINT_EDITAUTOMATION.can.return_value = True
+        st = PropertyMock(side_effect=[dict(), dict(b=2, c=3)])
+        mi = MagicMock()
+        type(mi).symtable = st
+        out = MagicMock()
+        out.read.return_value = "stdout"
+        err = MagicMock()
+        err.read.return_value = "stderr"
+
+        gi.return_value = (mi, out, err)
+        ge.return_value = "gcode"
+
+        rep = self.client.post(
+            "/automation/simulate",
+            data=dict(
+                automation=json.dumps([]),
+                symtable=json.dumps(dict(a=1, b=1)),
+            ),
+        )
+        self.assertEqual(rep.status_code, 200)
+        self.assertEqual(
+            json.loads(rep.get_data(as_text=True)),
+            {
+                "gcode": "gcode",
+                "stderr": "stderr",
+                "stdout": "stdout",
+                "symtable_diff": {"b": "2", "c": "3"},
+            },
+        )

continuousprint/automation.py

@@ -0,0 +1,45 @@
+from io import StringIO
+import re
+from asteval import Interpreter
+
+
+def getInterpreter(symbols):
+    out = StringIO()
+    err = StringIO()
+    interp = Interpreter(writer=out, err_writer=err)
+    # Merge in so default symbols (e.g. exceptions) are retained
+    for k, v in symbols.items():
+        interp.symtable[k] = v
+    return interp, out, err
+
+
+def genEventScript(automation: list, interp=None, logger=None) -> str:
+    result = []
+    for script, preprocessor in automation:
+        procval = True
+        if preprocessor is not None and preprocessor.strip() != "":
+            procval = interp(preprocessor)
+            if logger:
+                logger.info(
+                    f"EventHook preprocessor: {preprocessor}\nResult: {procval}"
+                )
+
+        if procval is None or procval is False:
+            continue
+        elif procval is True:
+            formatted = script
+        elif type(procval) is dict:
+            if logger:
+                logger.info(f"Appending script using formatting data {procval}")
+            formatted = script.format(**procval)
+        else:
+            raise Exception(
+                f"Invalid return type {type(procval)} for peprocessor {preprocessor}"
+            )
+
+        leftovers = re.findall(r"\{.*?\}", formatted)
+        if len(leftovers) > 0:
+            ppname = " (preprocessed)" if e.preprocessor is not None else ""
+            raise Exception(f"Unformatted placeholders in script{ppname}: {leftovers}")
+        result.append(formatted)
+    return "\n".join(result)

continuousprint/automation_test.py

@@ -0,0 +1,37 @@
+import unittest
+from .automation import getInterpreter, genEventScript
+
+
+class TestInterpreter(unittest.TestCase):
+    def testGetInterpreter(self):
+        interp, _, _ = getInterpreter(dict(a=1))
+        self.assertEqual(interp.symtable["a"], 1)
+
+
+class TestGenEventScript(unittest.TestCase):
+    def testEvalTrueFalseNone(self):
+        a = [("gcode1", "p1")]
+        self.assertEqual(genEventScript(a, lambda cond: True), "gcode1")
+        self.assertEqual(genEventScript(a, lambda cond: False), "")
+        self.assertEqual(genEventScript(a, lambda cond: None), "")
+
+    def testPlaceholderNoPreprocessor(self):
+        a = [("{foo} will never be formatted!", None)]
+        with self.assertRaises(Exception):
+            genEventScript(a, lambda cond: False)
+
+    def testEvalMissedPlaceholder(self):
+        a = [("{foo} will never be formatted!", "p1")]
+        with self.assertRaises(Exception):
+            genEventScript(a, lambda cond: dict(bar="baz"))
+
+    def testEvalFormat(self):
+        a = [("Hello {val}", "p1")]
+        self.assertEqual(
+            genEventScript(a, lambda cond: dict(val="World")), "Hello World"
+        )
+
+    def testEvalBadType(self):
+        a = [("dontcare", "p1")]
+        with self.assertRaises(Exception):
+            genEventScript(a, lambda cond: 7)

continuousprint/data/__init__.py

@@ -17,59 +17,119 @@
     )
 
 
+# This is used for running the preprocessor simulator in the settings page.
+SIMULATOR_DEFAULT_SYMTABLE = {
+    "current": {
+        "path": "testprint.gcode",
+        "materials": ["PLA_red_#ff0000"],
+        "bed_temp": 23.59,
+        "state": "printing",
+        "action": "TICK",
+    },
+    "external": {},
+    "metadata": {
+        "hash": "123abc",
+        "analysis": {
+            "printingArea": {
+                "maxX": 3,
+                "maxY": 6,
+                "maxZ": 9,
+                "minX": -3,
+                "minY": -6,
+                "minZ": -9.0,
+            },
+            "dimensions": {"depth": 5, "height": 10, "width": 15},
+            "estimatedPrintTime": 12345,
+            "filament": {"tool0": {"length": 123, "volume": 456}},
+        },
+        "continuousprint": {"profile": "Generic"},
+        "history": [
+            {
+                "timestamp": 1234567890,
+                "printTime": 100.0,
+                "success": True,
+                "printerProfile": "_default",
+            },
+        ],
+        "statistics": {
+            "averagePrintTime": {"_default": 100.0},
+            "lastPrintTime": {"_default": 100.0},
+        },
+    },
+}
+
+
 class CustomEvents(Enum):
     ACTIVATE = (
         "continuousprint_activate",
         "Queue Activated",
         "Fires when the queue is started, e.g. via the 'Start Managing' button.",
+        "inactive",
     )
     PRINT_START = (
         "continuousprint_start_print",
         "Print Start",
         "Fires when a new print is starting from the queue. Unlike OctoPrint events, this does not fire when event scripts are executed.",
+        "idle",
     )
     PRINT_SUCCESS = (
         "continuousprint_success",
         "Print Success",
         "Fires when the active print finishes. This will also fire for prints running before the queue was started. The final print will fire QUEUE_FINISH instead of PRINT_SUCCESS.",
+        "printing",
     )
     PRINT_CANCEL = (
         "continuousprint_cancel",
         "Print Cancel",
         "Fires when automation or the user has cancelled the active print.",
+        "printing",
     )
     COOLDOWN = (
         "continuousprint_cooldown",
         "Bed Cooldown",
         "Fires when bed cooldown is starting. Bed Cooldown is disabled by default - see the settings below.",
+        "idle",
     )
     FINISH = (
         "continuousprint_finish",
         "Queue Finished",
         "Fires when there is no work left to do and the plugin goes idle.",
+        "printing",
     )
     AWAITING_MATERIAL = (
         "continuousprint_awaiting_material",
         "Awaiting Material",
         "Fires once when the current job requires a different material than what is currently loaded. This requires SpoolManager to be installed (see Integrations).",
+        "idle",
     )
     DEACTIVATE = (
         "continuousprint_deactivate",
         "Queue Deactivated",
         "Fires when the queue is no longer actively managed. This script may be skipped if another print is underway when the queue goes inactive.",
+        "idle",
     )
 
-    def __init__(self, event, displayName, desc):
+    def __init__(self, event, displayName, desc, sym_state):
         self.event = event
         self.displayName = displayName
         self.desc = desc
+        self.sym_state = sym_state
+
+    @classmethod
+    def from_event(self, k):
+        evts = dict([(e.event, e) for e in self])
+        return evts[k]
 
     def as_dict(self):
-        return dict(event=self.event, display=self.displayName, desc=self.desc)
+        return dict(
+            event=self.event,
+            display=self.displayName,
+            desc=self.desc,
+            sym_state=self.sym_state,
+        )
 
 
 class Keys(Enum):
-
     BED_COOLDOWN_SCRIPT_DEPRECATED = (
         "cp_bed_cooldown_script",
         "; Put script to run before bed cools here\n",
@@ -97,6 +157,9 @@ class Keys(Enum):
     )  # One of "do_nothing", "add_draft", "add_printable"
     INFER_PROFILE = ("cp_infer_profile", True)
     AUTO_RECONNECT = ("cp_auto_reconnect", False)
+    SKIP_GCODE_COMMANDS = ("cp_skip_gcode_commands", "")
+    SLICER = ("cp_slicer", "")
+    SLICER_PROFILE = ("cp_slicer_profile", "")
 
     def __init__(self, setting, default):
         self.setting = setting
@@ -115,9 +178,11 @@ def __init__(self, setting, default):
         "js/continuousprint_api.js",
         "js/continuousprint_history_row.js",
         "js/continuousprint_set.js",
+        "js/continuousprint_stats.js",
         "js/continuousprint_job.js",
         "js/continuousprint_queue.js",
         "js/continuousprint_viewmodel.js",
+        "js/continuousprint_settings_event.js",
         "js/continuousprint_settings.js",
         "js/continuousprint.js",
     ],

continuousprint/data/data_test.py

@@ -52,7 +52,7 @@ def testcase(self):
             def runInterp(symtable):
                 stdout = StringIO()
                 interp = Interpreter(writer=stdout)
-                for (k, v) in symtable.items():
+                for k, v in symtable.items():
                     interp.symtable[k] = v
                 return interp(PREPROCESSORS[name]["body"], raise_errors=True), stdout