From 9023efbf13915c4bfbead09a0f155e98b737fcc4 Mon Sep 17 00:00:00 2001 From: Christopher Pickering Date: Tue, 25 Oct 2022 13:30:35 +0200 Subject: [PATCH] feat(backup): save backup file before sending to destinations closes #266 --- .prettierignore | 1 + runner/model.py | 2 -- runner/scripts/em_params.py | 2 +- runner/scripts/task_runner.py | 42 +++++++++++----------- scheduler/model.py | 2 -- setup.cfg | 3 ++ web/model.py | 2 -- web/static/fonts/inter/metadata.json | 2 +- web/static/fonts/inter/package.json | 2 +- web/static/fonts/rasa/files/file-list.json | 10 +----- web/static/fonts/rasa/metadata.json | 2 +- web/static/fonts/rasa/package.json | 2 +- web/static/js/search.js | 4 +-- web/web/table.py | 8 ++--- 14 files changed, 37 insertions(+), 47 deletions(-) diff --git a/.prettierignore b/.prettierignore index aa9f2c0b..156abbc6 100644 --- a/.prettierignore +++ b/.prettierignore @@ -6,3 +6,4 @@ node_modules/* web/static/lib/* web/static/fonts/* web/static/css/* +.lighthouserc/* diff --git a/runner/model.py b/runner/model.py index 770b15e5..fe091ee0 100644 --- a/runner/model.py +++ b/runner/model.py @@ -362,7 +362,6 @@ class ConnectionSftp(db.Model): username: Optional[str] = None key: Optional[str] = None password: Optional[str] = None - task: Optional[str] = None id = db.Column(db.Integer, primary_key=True, index=True) connection_id = db.Column( @@ -482,7 +481,6 @@ class ConnectionFtp(db.Model): path: Optional[str] = None username: Optional[str] = None password: Optional[str] = None - task: Optional[str] = None id = db.Column(db.Integer, primary_key=True, index=True) connection_id = db.Column( diff --git a/runner/scripts/em_params.py b/runner/scripts/em_params.py index 935a200a..dcc51e5a 100644 --- a/runner/scripts/em_params.py +++ b/runner/scripts/em_params.py @@ -28,7 +28,7 @@ def __load(self) -> Tuple[Dict[Any, Any], Dict[Any, Any]]: project_params = self.__parse_params(self.task.project.params) if self.task.params: - task_params = self.__parse_params(self.task.params) + task_params = self.__parse_params(self.task.params) # type: ignore return project_params, task_params diff --git a/runner/scripts/task_runner.py b/runner/scripts/task_runner.py index b2f6fb7d..ef6a62ab 100644 --- a/runner/scripts/task_runner.py +++ b/runner/scripts/task_runner.py @@ -636,6 +636,27 @@ def __store_files(self) -> None: f"Storing file {file_counter} of {len(self.source_files)}...", ) # store + # save historical copy + smb_path = Smb( + task=self.task, + run_id=self.run_id, + connection=None, # "default", + directory=self.temp_path, + ).save(overwrite=1, file_name=file_name) + + # log file details + db.session.add( + TaskFile( + name=file_name, + path=smb_path, + task_id=self.task.id, + job_id=self.run_id, + file_hash=file_hash, + size=file_size(str(os.path.getsize(file_path))), + ) + ) + db.session.commit() + # send to sftp if self.task.destination_sftp == 1 and self.task.destination_sftp_conn: if ( @@ -705,27 +726,6 @@ def __store_files(self) -> None: file_name=file_name, ) - # save historical copy - smb_path = Smb( - task=self.task, - run_id=self.run_id, - connection=None, # "default", - directory=self.temp_path, - ).save(overwrite=1, file_name=file_name) - - # log file details - db.session.add( - TaskFile( - name=file_name, - path=smb_path, - task_id=self.task.id, - job_id=self.run_id, - file_hash=file_hash, - size=file_size(str(os.path.getsize(file_path))), - ) - ) - db.session.commit() - def __send_email(self) -> None: logs = ( diff --git a/scheduler/model.py b/scheduler/model.py index 770b15e5..fe091ee0 100644 --- a/scheduler/model.py +++ b/scheduler/model.py @@ -362,7 +362,6 @@ class ConnectionSftp(db.Model): username: Optional[str] = None key: Optional[str] = None password: Optional[str] = None - task: Optional[str] = None id = db.Column(db.Integer, primary_key=True, index=True) connection_id = db.Column( @@ -482,7 +481,6 @@ class ConnectionFtp(db.Model): path: Optional[str] = None username: Optional[str] = None password: Optional[str] = None - task: Optional[str] = None id = db.Column(db.Integer, primary_key=True, index=True) connection_id = db.Column( diff --git a/setup.cfg b/setup.cfg index aba5590f..cd6f1b31 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,6 +14,9 @@ ignore_errors = True [mypy-*.ldap_auth] ignore_errors = True +[mypy-*.model] +disable_error_code = name-defined + [flake8] max-line-length = 99 per-file-ignores = diff --git a/web/model.py b/web/model.py index 770b15e5..fe091ee0 100644 --- a/web/model.py +++ b/web/model.py @@ -362,7 +362,6 @@ class ConnectionSftp(db.Model): username: Optional[str] = None key: Optional[str] = None password: Optional[str] = None - task: Optional[str] = None id = db.Column(db.Integer, primary_key=True, index=True) connection_id = db.Column( @@ -482,7 +481,6 @@ class ConnectionFtp(db.Model): path: Optional[str] = None username: Optional[str] = None password: Optional[str] = None - task: Optional[str] = None id = db.Column(db.Integer, primary_key=True, index=True) connection_id = db.Column( diff --git a/web/static/fonts/inter/metadata.json b/web/static/fonts/inter/metadata.json index ec0c8d49..d2885cf9 100644 --- a/web/static/fonts/inter/metadata.json +++ b/web/static/fonts/inter/metadata.json @@ -3,7 +3,7 @@ "defSubset": "latin", "fontId": "inter", "fontName": "Inter", - "lastModified": "2022-07-12", + "lastModified": "2022-09-22", "license": "https://fonts.google.com/attribution", "source": "https://fonts.google.com/", "styles": [ diff --git a/web/static/fonts/inter/package.json b/web/static/fonts/inter/package.json index d9e62a51..be3eb5ab 100644 --- a/web/static/fonts/inter/package.json +++ b/web/static/fonts/inter/package.json @@ -21,5 +21,5 @@ "type": "git", "url": "https://github.com/fontsource/fontsource.git" }, - "version": "4.5.12" + "version": "4.5.14" } diff --git a/web/static/fonts/rasa/files/file-list.json b/web/static/fonts/rasa/files/file-list.json index 53e205c3..dc782f6b 100644 --- a/web/static/fonts/rasa/files/file-list.json +++ b/web/static/fonts/rasa/files/file-list.json @@ -96,13 +96,5 @@ "./fonts/google/rasa/files/rasa-gujarati-variable-wghtOnly-italic.woff2", "./fonts/google/rasa/files/rasa-vietnamese-variable-wghtOnly-italic.woff2", "./fonts/google/rasa/files/rasa-latin-ext-variable-wghtOnly-italic.woff2", - "./fonts/google/rasa/files/rasa-latin-variable-wghtOnly-italic.woff2", - "./fonts/google/rasa/files/rasa-gujarati-variable-full-normal.woff2", - "./fonts/google/rasa/files/rasa-vietnamese-variable-full-normal.woff2", - "./fonts/google/rasa/files/rasa-latin-ext-variable-full-normal.woff2", - "./fonts/google/rasa/files/rasa-latin-variable-full-normal.woff2", - "./fonts/google/rasa/files/rasa-gujarati-variable-full-italic.woff2", - "./fonts/google/rasa/files/rasa-vietnamese-variable-full-italic.woff2", - "./fonts/google/rasa/files/rasa-latin-ext-variable-full-italic.woff2", - "./fonts/google/rasa/files/rasa-latin-variable-full-italic.woff2" + "./fonts/google/rasa/files/rasa-latin-variable-wghtOnly-italic.woff2" ] diff --git a/web/static/fonts/rasa/metadata.json b/web/static/fonts/rasa/metadata.json index d23d610b..a20e951e 100644 --- a/web/static/fonts/rasa/metadata.json +++ b/web/static/fonts/rasa/metadata.json @@ -3,7 +3,7 @@ "defSubset": "latin", "fontId": "rasa", "fontName": "Rasa", - "lastModified": "2022-07-19", + "lastModified": "2022-09-22", "license": "https://fonts.google.com/attribution", "source": "https://fonts.google.com/", "styles": [ diff --git a/web/static/fonts/rasa/package.json b/web/static/fonts/rasa/package.json index 8ed11f9e..d4d26861 100644 --- a/web/static/fonts/rasa/package.json +++ b/web/static/fonts/rasa/package.json @@ -21,5 +21,5 @@ "type": "git", "url": "https://github.com/fontsource/fontsource.git" }, - "version": "4.5.12" + "version": "4.5.13" } diff --git a/web/static/js/search.js b/web/static/js/search.js index 08a12031..2c36cc49 100644 --- a/web/static/js/search.js +++ b/web/static/js/search.js @@ -20,8 +20,8 @@ var q = new XMLHttpRequest(); searchResultsContainer = document.getElementById('search-results'); - if (!searchResultsContainer){ - return !1; + if (!searchResultsContainer) { + return !1; } // if data exists in localstorage, then enable search immediately. diff --git a/web/web/table.py b/web/web/table.py index 94db8c62..2bab5839 100644 --- a/web/web/table.py +++ b/web/web/table.py @@ -879,15 +879,15 @@ def task_log(task_id: int) -> Response: me.append({"empty_msg": "No log messages."}) if request.args.get("gte"): - logs = logs.filter(TaskLog.id >= request.args["gte"]).all() + logs = logs.filter(TaskLog.id >= request.args["gte"]) elif request.args.get("lt"): - logs = logs.filter(TaskLog.id < request.args["lt"]).limit(40).all() + logs = logs.filter(TaskLog.id < request.args["lt"]).limit(40) else: - logs = logs.limit(40).offset(0).all() + logs = logs.limit(40).offset(0) - for log in logs: + for log in logs.all(): log = dict(zip(cols.keys(), log))