From 121208a11dbf156e6f064f2f0936ffb22eeda0f5 Mon Sep 17 00:00:00 2001 From: binliu Date: Thu, 15 Dec 2022 11:47:41 +0000 Subject: [PATCH 1/5] add multi-thread unittest for mlflow handler Signed-off-by: binliu --- tests/test_handler_mlflow.py | 37 ++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tests/test_handler_mlflow.py b/tests/test_handler_mlflow.py index 3f43f97fbe..7939bad988 100644 --- a/tests/test_handler_mlflow.py +++ b/tests/test_handler_mlflow.py @@ -11,8 +11,10 @@ import glob import os +import shutil import tempfile import unittest +from concurrent.futures import ThreadPoolExecutor import numpy as np from ignite.engine import Engine, Events @@ -21,6 +23,29 @@ from monai.utils import path_to_uri +def dummy_train(tracking_folder): + tempdir = tempfile.mkdtemp() + + # set up engine + def _train_func(engine, batch): + return [batch + 1.0] + + engine = Engine(_train_func) + + # set up testing handler + test_path = os.path.join(tempdir, tracking_folder) + handler = MLFlowHandler( + iteration_log=False, + epoch_log=True, + tracking_uri=path_to_uri(test_path), + state_attributes=["test"], + close_on_complete=True, + ) + handler.attach(engine) + engine.run(range(3), max_epochs=2) + return test_path + + class TestHandlerMLFlow(unittest.TestCase): def test_metrics_track(self): experiment_param = {"backbone": "efficientnet_b0"} @@ -61,6 +86,18 @@ def _update_metric(engine): # check logging output self.assertTrue(len(glob.glob(test_path)) > 0) + def test_multi_thread(self): + test_uri_list = ["monai_mlflow_test1", "monai_mlflow_test2"] + with ThreadPoolExecutor(2, "Training") as executor: + futures = {} + for t in test_uri_list: + futures[t] = executor.submit(dummy_train, t) + + for _, future in futures.items(): + res = future.result() + self.assertTrue(len(glob.glob(res)) > 0) + shutil.rmtree(res) + if __name__ == "__main__": unittest.main() From 80b1134f8b5833ae9ebef27cbd8205fc0adc8a16 Mon Sep 17 00:00:00 2001 From: monai-bot Date: Thu, 15 Dec 2022 11:58:52 +0000 Subject: [PATCH 2/5] [MONAI] code formatting Signed-off-by: monai-bot --- tests/test_convert_data_type.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/test_convert_data_type.py b/tests/test_convert_data_type.py index aff4b7d995..43a0083256 100644 --- a/tests/test_convert_data_type.py +++ b/tests/test_convert_data_type.py @@ -31,7 +31,13 @@ for in_type in TEST_NDARRAYS_ALL + (int, float): for out_type in TEST_NDARRAYS_ALL: TESTS_LIST.append( - ([in_type(np.array(1.0)), in_type(np.array(1.0))], out_type(np.array([1.0, 1.0])), True, None, False) # type: ignore + ( + [in_type(np.array(1.0)), in_type(np.array(1.0))], + out_type(np.array([1.0, 1.0])), + True, + None, + False, + ) # type: ignore ) TESTS_LIST.append( ( @@ -44,7 +50,13 @@ ) if in_type is not float: TESTS_LIST.append( - ([in_type(np.array(257)), in_type(np.array(1))], out_type(np.array([255, 1])), True, np.uint8, True) # type: ignore + ( + [in_type(np.array(257)), in_type(np.array(1))], + out_type(np.array([255, 1])), + True, + np.uint8, + True, + ) # type: ignore ) TESTS_LIST.append( ( From 86aea0943118272ab2d7e3b651d2fdb523f3fd40 Mon Sep 17 00:00:00 2001 From: binliu Date: Thu, 15 Dec 2022 12:19:32 +0000 Subject: [PATCH 3/5] add a logic to avoid leave an unclosed folder Signed-off-by: binliu --- tests/test_handler_mlflow.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_handler_mlflow.py b/tests/test_handler_mlflow.py index 7939bad988..166588c0c1 100644 --- a/tests/test_handler_mlflow.py +++ b/tests/test_handler_mlflow.py @@ -95,8 +95,11 @@ def test_multi_thread(self): for _, future in futures.items(): res = future.result() - self.assertTrue(len(glob.glob(res)) > 0) - shutil.rmtree(res) + try: + self.assertTrue(len(glob.glob(res)) > 0) + finally: + if os.path.exists(res): + shutil.rmtree(res) if __name__ == "__main__": From d17da603dd9d37ffb65dc0282215420f5e0ef75c Mon Sep 17 00:00:00 2001 From: binliu Date: Thu, 15 Dec 2022 13:02:22 +0000 Subject: [PATCH 4/5] add setup and teardown function in mlflow handler to avoid unclosed folders Signed-off-by: binliu --- tests/test_handler_mlflow.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/test_handler_mlflow.py b/tests/test_handler_mlflow.py index 166588c0c1..2dfac344ae 100644 --- a/tests/test_handler_mlflow.py +++ b/tests/test_handler_mlflow.py @@ -47,6 +47,14 @@ def _train_func(engine, batch): class TestHandlerMLFlow(unittest.TestCase): + def setUp(self): + self.tmpdir_list = [] + + def tearDown(self) -> None: + for tmpdir in self.tmpdir_list: + if tmpdir and os.path.exists(tmpdir): + shutil.rmtree(tmpdir) + def test_metrics_track(self): experiment_param = {"backbone": "efficientnet_b0"} with tempfile.TemporaryDirectory() as tempdir: @@ -95,11 +103,8 @@ def test_multi_thread(self): for _, future in futures.items(): res = future.result() - try: - self.assertTrue(len(glob.glob(res)) > 0) - finally: - if os.path.exists(res): - shutil.rmtree(res) + self.tmpdir_list.append(res) + self.assertTrue(len(glob.glob(res)) > 0) if __name__ == "__main__": From 9fc4c40f5072c06a35542ee447b98dd5b24bca5c Mon Sep 17 00:00:00 2001 From: binliu Date: Thu, 15 Dec 2022 13:05:34 +0000 Subject: [PATCH 5/5] remove a redudant return type from mlflow handler Signed-off-by: binliu --- tests/test_handler_mlflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_handler_mlflow.py b/tests/test_handler_mlflow.py index 2dfac344ae..f41957840f 100644 --- a/tests/test_handler_mlflow.py +++ b/tests/test_handler_mlflow.py @@ -50,7 +50,7 @@ class TestHandlerMLFlow(unittest.TestCase): def setUp(self): self.tmpdir_list = [] - def tearDown(self) -> None: + def tearDown(self): for tmpdir in self.tmpdir_list: if tmpdir and os.path.exists(tmpdir): shutil.rmtree(tmpdir)