feat(backup): save backup file before sending to destinations

closes #266

.prettierignore

@@ -6,3 +6,4 @@ node_modules/*
 web/static/lib/*
 web/static/fonts/*
 web/static/css/*
+.lighthouserc/*

runner/model.py

@@ -362,7 +362,6 @@ class ConnectionSftp(db.Model):
     username: Optional[str] = None
     key: Optional[str] = None
     password: Optional[str] = None
-    task: Optional[str] = None
 
     id = db.Column(db.Integer, primary_key=True, index=True)
     connection_id = db.Column(
@@ -482,7 +481,6 @@ class ConnectionFtp(db.Model):
     path: Optional[str] = None
     username: Optional[str] = None
     password: Optional[str] = None
-    task: Optional[str] = None
 
     id = db.Column(db.Integer, primary_key=True, index=True)
     connection_id = db.Column(

runner/scripts/em_params.py

@@ -28,7 +28,7 @@ def __load(self) -> Tuple[Dict[Any, Any], Dict[Any, Any]]:
             project_params = self.__parse_params(self.task.project.params)
 
         if self.task.params:
-            task_params = self.__parse_params(self.task.params)
+            task_params = self.__parse_params(self.task.params)  # type: ignore
 
         return project_params, task_params
 

runner/scripts/task_runner.py

@@ -636,6 +636,27 @@ def __store_files(self) -> None:
                     f"Storing file {file_counter} of {len(self.source_files)}...",
                 )
             # store
+            # save historical copy
+            smb_path = Smb(
+                task=self.task,
+                run_id=self.run_id,
+                connection=None,  # "default",
+                directory=self.temp_path,
+            ).save(overwrite=1, file_name=file_name)
+
+            # log file details
+            db.session.add(
+                TaskFile(
+                    name=file_name,
+                    path=smb_path,
+                    task_id=self.task.id,
+                    job_id=self.run_id,
+                    file_hash=file_hash,
+                    size=file_size(str(os.path.getsize(file_path))),
+                )
+            )
+            db.session.commit()
+
             # send to sftp
             if self.task.destination_sftp == 1 and self.task.destination_sftp_conn:
                 if (
@@ -705,27 +726,6 @@ def __store_files(self) -> None:
                         file_name=file_name,
                     )
 
-            # save historical copy
-            smb_path = Smb(
-                task=self.task,
-                run_id=self.run_id,
-                connection=None,  # "default",
-                directory=self.temp_path,
-            ).save(overwrite=1, file_name=file_name)
-
-            # log file details
-            db.session.add(
-                TaskFile(
-                    name=file_name,
-                    path=smb_path,
-                    task_id=self.task.id,
-                    job_id=self.run_id,
-                    file_hash=file_hash,
-                    size=file_size(str(os.path.getsize(file_path))),
-                )
-            )
-            db.session.commit()
-
     def __send_email(self) -> None:
 
         logs = (

scheduler/model.py

@@ -362,7 +362,6 @@ class ConnectionSftp(db.Model):
     username: Optional[str] = None
     key: Optional[str] = None
     password: Optional[str] = None
-    task: Optional[str] = None
 
     id = db.Column(db.Integer, primary_key=True, index=True)
     connection_id = db.Column(
@@ -482,7 +481,6 @@ class ConnectionFtp(db.Model):
     path: Optional[str] = None
     username: Optional[str] = None
     password: Optional[str] = None
-    task: Optional[str] = None
 
     id = db.Column(db.Integer, primary_key=True, index=True)
     connection_id = db.Column(

setup.cfg

@@ -14,6 +14,9 @@ ignore_errors = True
 [mypy-*.ldap_auth]
 ignore_errors = True
 
+[mypy-*.model]
+disable_error_code = name-defined
+
 [flake8]
 max-line-length = 99
 per-file-ignores =

web/model.py

@@ -362,7 +362,6 @@ class ConnectionSftp(db.Model):
     username: Optional[str] = None
     key: Optional[str] = None
     password: Optional[str] = None
-    task: Optional[str] = None
 
     id = db.Column(db.Integer, primary_key=True, index=True)
     connection_id = db.Column(
@@ -482,7 +481,6 @@ class ConnectionFtp(db.Model):
     path: Optional[str] = None
     username: Optional[str] = None
     password: Optional[str] = None
-    task: Optional[str] = None
 
     id = db.Column(db.Integer, primary_key=True, index=True)
     connection_id = db.Column(

web/static/fonts/inter/metadata.json

@@ -3,7 +3,7 @@
   "defSubset": "latin",
   "fontId": "inter",
   "fontName": "Inter",
-  "lastModified": "2022-07-12",
+  "lastModified": "2022-09-22",
   "license": "https://fonts.google.com/attribution",
   "source": "https://fonts.google.com/",
   "styles": [

web/static/fonts/inter/package.json

@@ -21,5 +21,5 @@
     "type": "git",
     "url": "https://github.com/fontsource/fontsource.git"
   },
-  "version": "4.5.12"
+  "version": "4.5.14"
 }

web/static/fonts/rasa/files/file-list.json

@@ -96,13 +96,5 @@
   "./fonts/google/rasa/files/rasa-gujarati-variable-wghtOnly-italic.woff2",
   "./fonts/google/rasa/files/rasa-vietnamese-variable-wghtOnly-italic.woff2",
   "./fonts/google/rasa/files/rasa-latin-ext-variable-wghtOnly-italic.woff2",
-  "./fonts/google/rasa/files/rasa-latin-variable-wghtOnly-italic.woff2",
-  "./fonts/google/rasa/files/rasa-gujarati-variable-full-normal.woff2",
-  "./fonts/google/rasa/files/rasa-vietnamese-variable-full-normal.woff2",
-  "./fonts/google/rasa/files/rasa-latin-ext-variable-full-normal.woff2",
-  "./fonts/google/rasa/files/rasa-latin-variable-full-normal.woff2",
-  "./fonts/google/rasa/files/rasa-gujarati-variable-full-italic.woff2",
-  "./fonts/google/rasa/files/rasa-vietnamese-variable-full-italic.woff2",
-  "./fonts/google/rasa/files/rasa-latin-ext-variable-full-italic.woff2",
-  "./fonts/google/rasa/files/rasa-latin-variable-full-italic.woff2"
+  "./fonts/google/rasa/files/rasa-latin-variable-wghtOnly-italic.woff2"
 ]

web/static/fonts/rasa/metadata.json

@@ -3,7 +3,7 @@
   "defSubset": "latin",
   "fontId": "rasa",
   "fontName": "Rasa",
-  "lastModified": "2022-07-19",
+  "lastModified": "2022-09-22",
   "license": "https://fonts.google.com/attribution",
   "source": "https://fonts.google.com/",
   "styles": [

web/static/fonts/rasa/package.json

@@ -21,5 +21,5 @@
     "type": "git",
     "url": "https://github.com/fontsource/fontsource.git"
   },
-  "version": "4.5.12"
+  "version": "4.5.13"
 }

web/static/js/search.js

@@ -20,8 +20,8 @@
     var q = new XMLHttpRequest();
 
     searchResultsContainer = document.getElementById('search-results');
-    if (!searchResultsContainer){
-        return !1;
+    if (!searchResultsContainer) {
+      return !1;
     }
 
     // if data exists in localstorage, then enable search immediately.

web/web/table.py

@@ -879,15 +879,15 @@ def task_log(task_id: int) -> Response:
     me.append({"empty_msg": "No log messages."})
 
     if request.args.get("gte"):
-        logs = logs.filter(TaskLog.id >= request.args["gte"]).all()
+        logs = logs.filter(TaskLog.id >= request.args["gte"])
 
     elif request.args.get("lt"):
-        logs = logs.filter(TaskLog.id < request.args["lt"]).limit(40).all()
+        logs = logs.filter(TaskLog.id < request.args["lt"]).limit(40)
 
     else:
-        logs = logs.limit(40).offset(0).all()
+        logs = logs.limit(40).offset(0)
 
-    for log in logs:
+    for log in logs.all():
 
         log = dict(zip(cols.keys(), log))