Merge pull request #1048 from sbrunner/improve-status

Improve the status page

.prospector.yaml

@@ -28,6 +28,7 @@ pylint:
     - too-many-nested-blocks
     - too-many-arguments
     - too-many-return-statements
+    - too-many-instance-attributes
     - wrong-import-order
     - line-too-long
     - no-value-for-parameter

scan_to_paperless/process.py

@@ -1665,12 +1665,10 @@ def _process(
     with open(config_file_name, encoding="utf-8") as config_file:
         config: schema.Configuration = yaml.load(config_file.read())
     if config is None:
-        status.set_status(config_file_name, "Empty config")
         print_waiting = True
         return dirty, print_waiting
 
     if not is_sources_present(config["images"], root_folder):
-        status.set_status(config_file_name, "Missing images")
         print_waiting = True
         return dirty, print_waiting
 
@@ -1686,7 +1684,6 @@ def _process(
             save_config(config, config_file_name)
             if os.path.exists(os.path.join(root_folder, "REMOVE_TO_CONTINUE")):
                 os.remove(os.path.join(root_folder, "REMOVE_TO_CONTINUE"))
-            status.set_status(config_file_name, "Rerun step")
             print_waiting = True
             rerun = True
 
@@ -1701,16 +1698,11 @@ def _process(
         if is_sources_present(step["sources"], root_folder):
             if not disable_remove_to_continue:
                 if os.path.exists(os.path.join(root_folder, "REMOVE_TO_CONTINUE")) and not rerun:
-                    status.set_status(
-                        config_file_name,
-                        scan_to_paperless.status.WAITING_STATUS_NAME,
-                        scan_to_paperless.status.WAITING_STATUS_DESCRIPTION,
-                    )
                     return dirty, print_waiting
             if os.path.exists(os.path.join(root_folder, "DONE")) and not rerun:
-                status.set_status(config_file_name, "Done")
                 return dirty, print_waiting
 
+            status.set_current_config(config_file_name)
             status.set_status(config_file_name, "Processing")
             print_waiting = True
             dirty = True
@@ -1735,19 +1727,13 @@ def _process(
                     config["steps"].append(next_step)
                 save_config(config, config_file_name)
                 if done:
-                    status.set_status(config_file_name, "Done")
                     with open(os.path.join(root_folder, "DONE"), "w", encoding="utf-8"):
                         pass
                 elif not disable_remove_to_continue:
-                    status.set_status(
-                        config_file_name,
-                        scan_to_paperless.status.WAITING_STATUS_NAME,
-                        scan_to_paperless.status.WAITING_STATUS_DESCRIPTION,
-                    )
                     with open(os.path.join(root_folder, "REMOVE_TO_CONTINUE"), "w", encoding="utf-8"):
                         pass
-        else:
-            status.set_status(config_file_name, "Missing sources")
+            status.set_current_folder(None)
+            status.write()
 
     except Exception as exception:
         print(exception)
@@ -1771,7 +1757,6 @@ def _process(
                 yaml.dump(out, error_file)
             stream = ruamel.yaml.compat.StringIO()
             yaml.dump(out, stream)
-            status.set_status(config_file_name, "Error", "<pre>" + stream.getvalue() + "</pre>")
         except Exception as exception2:
             print(exception2)
             print(traceback.format_exc())
@@ -1781,7 +1766,6 @@ def _process(
                 yaml.dump(out, error_file)
             stream = ruamel.yaml.compat.StringIO()
             yaml.dump(out, stream)
-            status.set_status(config_file_name, "Error", "<pre>" + stream.getvalue() + "</pre>")
     return dirty, print_waiting
 
 

scan_to_paperless/status.py

@@ -4,7 +4,7 @@
 import glob
 import html
 import os.path
-from typing import Dict, NamedTuple
+from typing import Dict, NamedTuple, Optional
 
 from ruamel.yaml.main import YAML
 
@@ -34,16 +34,35 @@ def __init__(self, no_write: bool = False) -> None:
         self._file = os.path.join(os.environ.get("SCAN_SOURCE_FOLDER", "/source"), "status.html")
         self._status: Dict[str, _Folder] = {}
         self._global_status = "Starting..."
+        self._global_status_update = datetime.datetime.utcnow().replace(microsecond=0)
         self._start_time = datetime.datetime.utcnow().replace(microsecond=0)
         self._last_scan = datetime.datetime.utcnow()
+        self._current_folder: Optional[str] = None
         self.scan()
 
     def set_global_status(self, status: str) -> None:
         """Set the global status."""
 
+        if self._global_status == status:
+            return
+
         self._global_status = status
+        self._global_status_update = datetime.datetime.utcnow().replace(microsecond=0)
+
         self.write()
 
+    def set_current_folder(self, name: Optional[str]) -> None:
+        """Set the current folder."""
+
+        self._current_folder = name
+
+    def set_current_config(self, name: str) -> None:
+        """Set the current config file."""
+
+        if name.endswith("/config.yaml"):
+            name = os.path.basename(os.path.dirname(name))
+        self._current_folder = name
+
     def set_status(self, name: str, status: str, details: str = "") -> None:
         """Set the status of a folder."""
 
@@ -61,8 +80,8 @@ def set_status(self, name: str, status: str, details: str = "") -> None:
     def scan(self) -> None:
         """Scan for changes for waiting documents."""
 
-        for name, folder in self._status.items():
-            if folder.status == WAITING_STATUS_NAME:
+        for name in self._status:
+            if name != self._current_folder:
                 self._update_status(name)
 
         names = []
@@ -74,6 +93,23 @@ def scan(self) -> None:
             if name not in self._status:
                 self._update_status(name, force=True)
 
+        for folder_name in glob.glob(os.path.join(os.environ.get("SCAN_SOURCE_FOLDER", "/source"), "*")):
+            name = os.path.basename(folder_name)
+
+            if name not in self._status:
+                names.append(name)
+
+                self.set_status(
+                    name,
+                    "Missing config",
+                    ", ".join(
+                        glob.glob(
+                            os.path.join(os.environ.get("SCAN_SOURCE_FOLDER", "/source"), folder_name, "**"),
+                            recursive=True,
+                        )
+                    ),
+                )
+
         for name in self._status:  # pylint: disable=consider-using-dict-items
             if name not in names:
                 del self._status[name]
@@ -89,14 +125,23 @@ def _update_status(self, name: str, force: bool = False) -> None:
             ) as error_file:
                 error = yaml.load(error_file)
 
-            self.set_status(name, "Error: " + error["error"])
+            self.set_status(
+                name, "Error: " + error["error"], "<code>" + "<br />".join(error["traceback"]) + "</code>"
+            )
 
         with open(
             os.path.join(os.environ.get("SCAN_SOURCE_FOLDER", "/source"), name, "config.yaml"),
             encoding="utf-8",
         ) as config_file:
             config = yaml.load(config_file)
 
+        if config is None:
+            self.set_status(name, "Empty config")
+            return
+
+        if os.path.exists(os.path.join(os.environ.get("SCAN_SOURCE_FOLDER", "/source"), name, "DONE")):
+            self.set_status(name, "Done")
+
         if os.path.exists(
             os.path.join(os.environ.get("SCAN_SOURCE_FOLDER", "/source"), name, "REMOVE_TO_CONTINUE")
         ):
@@ -160,7 +205,9 @@ def write(self) -> None:
   </head>
   <body class="px-5 py-4">
     <h1>Scan to Paperless status</h1>
-    <p>{self._global_status}</p>
+    <p>{self._global_status} since <script>
+    window.document.write(new Date('{self._global_status_update.isoformat()}Z').toLocaleString());
+    </script></p>
     <p>Started at: <script>
     window.document.write(new Date('{self._start_time.isoformat()}Z').toLocaleString());
     </script></p>
@@ -180,8 +227,9 @@ def write(self) -> None:
 """
             )
             for name, folder in self._status.items():
+                tr_attributes = ' class="alert alert-info"' if name == self._current_folder else ""
                 status_file.write(
-                    f"""        <tr>
+                    f"""        <tr{tr_attributes}>
           <td><a href="./{name}" target="_blank">{name}</a></td>
           <td>{folder.status}</td>
           <td>{folder.details}</td>