From 9c5192dd8e9fc18b19b069619f6a00f05c385ad9 Mon Sep 17 00:00:00 2001 From: Zhaosi Qu Date: Mon, 8 May 2023 00:32:39 +0300 Subject: [PATCH] Fix bugs --- core/Dockerfile | 2 ++ core/functions/project/prescribe.py | 7 +++-- core/services/event_log.py | 3 ++- core/services/project.py | 2 ++ .../causallift_resource_allocation/Dockerfile | 2 ++ .../causallift_treatment_effect/Dockerfile | 2 ++ .../causallift_treatment_effect/algorithm.py | 27 ++++++++++++------- plugins/knn_next_activity/Dockerfile | 2 ++ plugins/random_forest_alarm/Dockerfile | 2 ++ processor/Dockerfile | 2 ++ 10 files changed, 39 insertions(+), 12 deletions(-) diff --git a/core/Dockerfile b/core/Dockerfile index 5e6314f..5120cf1 100644 --- a/core/Dockerfile +++ b/core/Dockerfile @@ -3,4 +3,6 @@ WORKDIR /code COPY ./core /code/core COPY ./simulator /code/simulator RUN pip install --no-cache-dir -r core/requirements.txt +ENV PYTHONUNBUFFERED=1 +ENV PYTHONIOENCODING=UTF-8 CMD ["uvicorn", "core.main:app", "--host", "0.0.0.0", "--port", "8000", "--proxy-headers"] diff --git a/core/functions/project/prescribe.py b/core/functions/project/prescribe.py index b9eb59f..d5d565d 100644 --- a/core/functions/project/prescribe.py +++ b/core/functions/project/prescribe.py @@ -26,7 +26,7 @@ logger = logging.getLogger(__name__) -def get_ongoing_dataset_result_key(file: BinaryIO, extension: str, seperator: str, +def get_ongoing_dataset_result_key(file: BinaryIO | bytes, extension: str, seperator: str, db_project: project_model.Project) -> str: # Get the result key of the ongoing dataset result = "" @@ -34,7 +34,10 @@ def get_ongoing_dataset_result_key(file: BinaryIO, extension: str, seperator: st try: temp_path = get_new_path(f"{path.TEMP_PATH}/", suffix=f".{extension}") with open(temp_path, "wb") as f: - f.write(file.read()) + if isinstance(file, bytes): + f.write(file) + else: + f.write(file.read()) # Get dataframe from file try: diff --git a/core/services/event_log.py b/core/services/event_log.py index bf87f5e..daec071 100644 --- a/core/services/event_log.py +++ b/core/services/event_log.py @@ -48,9 +48,10 @@ def process_uploaded_event_log(file: UploadFile, separator: str, test: UploadFil # Save test file to memory if test and test.file: + content = test.file.read() memory.log_tests[db_event_log.id] = { "date": datetime.now(), - "file": deepcopy(test.file), + "file": content, "extension": get_extension(test.filename), "separator": separator } diff --git a/core/services/project.py b/core/services/project.py index fa34d6b..676e431 100644 --- a/core/services/project.py +++ b/core/services/project.py @@ -251,6 +251,8 @@ def process_ongoing_dataset_result(project_id: int, result_key: str, background_ logger.warning("Start to merge the result") for plugin_result in result["results"].values(): for case_id, case_result in plugin_result.items(): + if case_id not in result["cases"]: + continue result["cases"][case_id]["prescriptions"].append(case_result) logger.warning("Merge the result successfully") diff --git a/plugins/causallift_resource_allocation/Dockerfile b/plugins/causallift_resource_allocation/Dockerfile index 036477c..21c67e5 100644 --- a/plugins/causallift_resource_allocation/Dockerfile +++ b/plugins/causallift_resource_allocation/Dockerfile @@ -3,4 +3,6 @@ WORKDIR /code COPY ./core /code/core COPY ./plugins /code/plugins RUN pip install --no-cache-dir -r plugins/causallift_resource_allocation/requirements.txt +ENV PYTHONUNBUFFERED=1 +ENV PYTHONIOENCODING=UTF-8 CMD ["python", "-m", "plugins.causallift_resource_allocation.main"] diff --git a/plugins/causallift_treatment_effect/Dockerfile b/plugins/causallift_treatment_effect/Dockerfile index 58103d9..d84d47b 100644 --- a/plugins/causallift_treatment_effect/Dockerfile +++ b/plugins/causallift_treatment_effect/Dockerfile @@ -3,4 +3,6 @@ WORKDIR /code COPY ./core /code/core COPY ./plugins /code/plugins RUN pip install --no-cache-dir -r plugins/causallift_treatment_effect/requirements.txt +ENV PYTHONUNBUFFERED=1 +ENV PYTHONIOENCODING=UTF-8 CMD ["python", "-m", "plugins.causallift_treatment_effect.main"] diff --git a/plugins/causallift_treatment_effect/algorithm.py b/plugins/causallift_treatment_effect/algorithm.py index 64f01a5..fd65d17 100644 --- a/plugins/causallift_treatment_effect/algorithm.py +++ b/plugins/causallift_treatment_effect/algorithm.py @@ -113,15 +113,24 @@ def predict_df(self, df: DataFrame) -> dict: # Get the result for each length threads = [] - for length, test_df in test_dfs.items(): - training_df = self.get_data()["training_dfs"].get(length) - if training_df is None: - continue - t = Thread(target=self.get_result_thread, args=(self, result_dfs, length, training_df, test_df)) - t.start() - threads.append(t) - for t in threads: - t.join() + count_of_length = len(test_dfs) + if count_of_length <= 50: + for length, test_df in test_dfs.items(): + training_df = self.get_data()["training_dfs"].get(length) + if training_df is None: + continue + t = Thread(target=self.get_result_thread, args=(self, result_dfs, length, training_df, test_df)) + t.start() + threads.append(t) + for t in threads: + t.join() + else: + for length, test_df in test_dfs.items(): + training_df = self.get_data()["training_dfs"].get(length) + if training_df is None: + continue + result_df = self.get_result(training_df, test_df) + result_dfs[length] = result_df # Merge the result if len(result_dfs) <= 0: diff --git a/plugins/knn_next_activity/Dockerfile b/plugins/knn_next_activity/Dockerfile index 6d20369..e139ebb 100644 --- a/plugins/knn_next_activity/Dockerfile +++ b/plugins/knn_next_activity/Dockerfile @@ -3,4 +3,6 @@ WORKDIR /code COPY ./core /code/core COPY ./plugins /code/plugins RUN pip install --no-cache-dir -r plugins/knn_next_activity/requirements.txt +ENV PYTHONUNBUFFERED=1 +ENV PYTHONIOENCODING=UTF-8 CMD ["python", "-m", "plugins.knn_next_activity.main"] diff --git a/plugins/random_forest_alarm/Dockerfile b/plugins/random_forest_alarm/Dockerfile index 5e95a25..4bce8fa 100644 --- a/plugins/random_forest_alarm/Dockerfile +++ b/plugins/random_forest_alarm/Dockerfile @@ -3,4 +3,6 @@ WORKDIR /code COPY ./core /code/core COPY ./plugins /code/plugins RUN pip install --no-cache-dir -r plugins/random_forest_alarm/requirements.txt +ENV PYTHONUNBUFFERED=1 +ENV PYTHONIOENCODING=UTF-8 CMD ["python", "-m", "plugins.random_forest_alarm.main"] diff --git a/processor/Dockerfile b/processor/Dockerfile index 571ebc3..21958b6 100644 --- a/processor/Dockerfile +++ b/processor/Dockerfile @@ -3,4 +3,6 @@ WORKDIR /code COPY ./core /code/core COPY ./processor /code/processor RUN pip install --no-cache-dir -r processor/requirements.txt +ENV PYTHONUNBUFFERED=1 +ENV PYTHONIOENCODING=UTF-8 CMD ["python", "-m", "processor.main"]