From dee0e71a7491ca59c0349bb9b84a81d8e2ff2d9c Mon Sep 17 00:00:00 2001 From: simonrempel Date: Thu, 14 Jul 2022 10:55:47 +0200 Subject: [PATCH 1/5] Adds monitoring of upload task --- refinery/__init__.py | 54 ++++++++++++++++++++++++++++++++++++++++---- refinery/settings.py | 4 ++++ 2 files changed, 53 insertions(+), 5 deletions(-) diff --git a/refinery/__init__.py b/refinery/__init__.py index 036d372..4f0df48 100644 --- a/refinery/__init__.py +++ b/refinery/__init__.py @@ -8,6 +8,8 @@ import os.path from tqdm import tqdm import spacy +import time +from refinery import settings class Client: @@ -148,7 +150,9 @@ def get_record_export( ) else: - msg.warn("There are no attributes that can be tokenized in this project.") + msg.warn( + "There are no attributes that can be tokenized in this project." + ) if download_to is not None: df.to_json(download_to, orient="records") @@ -214,8 +218,48 @@ def post_file_import( file_name, ) if success: - msg.good(f"Uploaded {path} to your project.") - return True + msg.good(f"Uploaded {path} to object storage.") + upload_task_id = ( + upload_task_id.split("/")[-1] + if "/" in upload_task_id + else upload_task_id + ) + self.__monitor_task(upload_task_id) + else: - msg.fail(f"Could not upload {path} to your project.") - return False + msg_text = f"Could not upload {path} to your project." + msg.fail(msg_text) + raise exceptions.FileImportError(msg_text) + + def __monitor_task(self, upload_task_id): + do_monitoring = True + idx = 0 + last_progress = 0.0 + with tqdm( + total=100.00, + colour="green", + bar_format="{desc}: {percentage:.2f}%|{bar:10}| {n:.2f}/{total_fmt}", + ) as pbar: + pbar.set_description_str(desc="PENDING", refresh=True) + while do_monitoring: + idx += 1 + task = self.__get_task(upload_task_id) + task_progress = task.get("progress") if task.get("progress") else 0.0 + task_state = task.get("state") if task.get("state") else "FAILED" + progress = task_progress - last_progress + last_progress = task_progress + pbar.update(progress) + pbar.set_description_str(desc=task_state, refresh=True) + if task_state == "DONE" or task_state == "FAILED": + do_monitoring = False + if idx >= 100: + raise exceptions.FileImportError( + "Timeout while upload, please check the upload progress in the UI." + ) + time.sleep(0.5) + + def __get_task(self, upload_task_id): + api_response = api_calls.get_request( + settings.get_task(self.project_id, upload_task_id), self.session_token + ) + return api_response diff --git a/refinery/settings.py b/refinery/settings.py index 0adcb50..64c7c5a 100644 --- a/refinery/settings.py +++ b/refinery/settings.py @@ -46,3 +46,7 @@ def get_import_url(project_id: str) -> str: def get_base_config(project_id: str) -> str: return f"{get_project_url(project_id)}/import/base_config" + + +def get_task(project_id: str, task_id: str) -> str: + return f"{get_project_url(project_id)}/import/task/{task_id}" From 380932026943cc42043d4cb8919649168333217e Mon Sep 17 00:00:00 2001 From: simonrempel Date: Thu, 14 Jul 2022 10:56:40 +0200 Subject: [PATCH 2/5] Adds typing --- refinery/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/refinery/__init__.py b/refinery/__init__.py index 4f0df48..24dcbe7 100644 --- a/refinery/__init__.py +++ b/refinery/__init__.py @@ -231,7 +231,7 @@ def post_file_import( msg.fail(msg_text) raise exceptions.FileImportError(msg_text) - def __monitor_task(self, upload_task_id): + def __monitor_task(self, upload_task_id: str) -> None: do_monitoring = True idx = 0 last_progress = 0.0 @@ -258,7 +258,7 @@ def __monitor_task(self, upload_task_id): ) time.sleep(0.5) - def __get_task(self, upload_task_id): + def __get_task(self, upload_task_id: str) -> Dict[str, Any]: api_response = api_calls.get_request( settings.get_task(self.project_id, upload_task_id), self.session_token ) From 58d36238097feb7e985cf502cd2f7937b75f15c2 Mon Sep 17 00:00:00 2001 From: simonrempel Date: Thu, 14 Jul 2022 11:07:40 +0200 Subject: [PATCH 3/5] Fixes typing error --- refinery/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/refinery/__init__.py b/refinery/__init__.py index 24dcbe7..f1822ef 100644 --- a/refinery/__init__.py +++ b/refinery/__init__.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- +from black import Any from wasabi import msg import pandas as pd from refinery import authentication, api_calls, settings, exceptions, util From 1cf01811141ace22ee17ee989f225e184507db25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20H=C3=B6tter?= Date: Thu, 14 Jul 2022 12:32:40 +0200 Subject: [PATCH 4/5] adds print statements to upload --- refinery/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/refinery/__init__.py b/refinery/__init__.py index f1822ef..f31cd7e 100644 --- a/refinery/__init__.py +++ b/refinery/__init__.py @@ -252,6 +252,10 @@ def __monitor_task(self, upload_task_id: str) -> None: pbar.update(progress) pbar.set_description_str(desc=task_state, refresh=True) if task_state == "DONE" or task_state == "FAILED": + if task_state == "DONE": + msg.good("Upload successful.") + else: + msg.fail("Upload failed. Please look into the UI notification center for more details.") do_monitoring = False if idx >= 100: raise exceptions.FileImportError( From 48ba5d13e18f8cf609d3630261d17cd0c425ad17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20H=C3=B6tter?= Date: Thu, 14 Jul 2022 12:35:10 +0200 Subject: [PATCH 5/5] re-arrange prints --- refinery/__init__.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/refinery/__init__.py b/refinery/__init__.py index f31cd7e..6e70e9a 100644 --- a/refinery/__init__.py +++ b/refinery/__init__.py @@ -236,6 +236,7 @@ def __monitor_task(self, upload_task_id: str) -> None: do_monitoring = True idx = 0 last_progress = 0.0 + print_success_message = False with tqdm( total=100.00, colour="green", @@ -252,16 +253,17 @@ def __monitor_task(self, upload_task_id: str) -> None: pbar.update(progress) pbar.set_description_str(desc=task_state, refresh=True) if task_state == "DONE" or task_state == "FAILED": - if task_state == "DONE": - msg.good("Upload successful.") - else: - msg.fail("Upload failed. Please look into the UI notification center for more details.") + print_success_message = task_state == "DONE" do_monitoring = False if idx >= 100: raise exceptions.FileImportError( "Timeout while upload, please check the upload progress in the UI." ) time.sleep(0.5) + if print_success_message: + msg.good("File upload successful.") + else: + msg.fail("Upload failed. Please look into the UI notification center for more details.") def __get_task(self, upload_task_id: str) -> Dict[str, Any]: api_response = api_calls.get_request(