From 18c4efcbff10941b1e3613045b16656e57e4be12 Mon Sep 17 00:00:00 2001 From: Mingxin Zheng <18563433+mingxin-zheng@users.noreply.github.com> Date: Tue, 18 Oct 2022 17:07:09 +0800 Subject: [PATCH 1/5] change datasets from task 09 to 04 for all auto3dseg notebook Signed-off-by: Mingxin Zheng <18563433+mingxin-zheng@users.noreply.github.com> --- auto3dseg/README.md | 1 + .../auto3dseg_autorunner_ref_api.ipynb | 111 ++- .../notebooks/auto3dseg_hello_world.ipynb | 350 ++++----- auto3dseg/notebooks/auto_runner.ipynb | 93 +-- auto3dseg/notebooks/ensemble_byoc.ipynb | 683 +++++++++--------- auto3dseg/notebooks/hpo_nni.ipynb | 8 +- auto3dseg/notebooks/hpo_optuna.ipynb | 344 +++++---- .../tasks/msd/Task04_Hippocampus/README.md | 3 + .../tasks/msd/Task04_Hippocampus/input.yaml | 10 + .../msd_task04_hippocampus_folds.json | 204 ++++++ 10 files changed, 992 insertions(+), 815 deletions(-) create mode 100644 auto3dseg/tasks/msd/Task04_Hippocampus/README.md create mode 100644 auto3dseg/tasks/msd/Task04_Hippocampus/input.yaml create mode 100644 auto3dseg/tasks/msd/Task04_Hippocampus/msd_task04_hippocampus_folds.json diff --git a/auto3dseg/README.md b/auto3dseg/README.md index 565899d1bf..83032c7da7 100644 --- a/auto3dseg/README.md +++ b/auto3dseg/README.md @@ -43,6 +43,7 @@ We have demonstrated preliminary results of public datasets are described in the - [INSTANCE22](tasks/instance22) - [Beyond the Cranial Vault (BTCV) Abdomen Dataset](tasks/btcv) - Medical Segmentation Decathlon (MSD) Dataset + - [Task04 Task04_Hippocampus](tasks/msd/Task04_Hippocampus) - [Task05 Prostate](tasks/msd/Task05_Prostate) - [Task09 Spleen](tasks/msd/Task09_Spleen) diff --git a/auto3dseg/notebooks/auto3dseg_autorunner_ref_api.ipynb b/auto3dseg/notebooks/auto3dseg_autorunner_ref_api.ipynb index 4a4e921601..ad9ebfe1a2 100644 --- a/auto3dseg/notebooks/auto3dseg_autorunner_ref_api.ipynb +++ b/auto3dseg/notebooks/auto3dseg_autorunner_ref_api.ipynb @@ -24,7 +24,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -40,13 +40,15 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "import os\n", "import torch\n", "\n", + "from pathlib import Path\n", + "\n", "from monai.apps import download_and_extract\n", "from monai.apps.auto3dseg import (\n", " DataAnalyzer,\n", @@ -69,33 +71,26 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Task09_Spleen.tar: 1.50GB [00:43, 36.7MB/s] \n" - ] - }, { "name": "stdout", "output_type": "stream", "text": [ - "2022-09-29 03:49:02,357 - INFO - Downloaded: Task09_Spleen.tar\n", - "2022-09-29 03:49:02,359 - INFO - Expected md5 is None, skip md5 check for file Task09_Spleen.tar.\n", - "2022-09-29 03:49:02,359 - INFO - Writing into directory: ./.\n" + "2022-10-18 08:14:33,395 - INFO - Expected md5 is None, skip md5 check for file Task04_Hippocampus.tar.\n", + "2022-10-18 08:14:33,395 - INFO - File exists: Task04_Hippocampus.tar, skipped downloading.\n", + "2022-10-18 08:14:33,396 - INFO - Non-empty folder exists in Task04_Hippocampus, skipped extracting.\n" ] } ], "source": [ - "root = \"./\"\n", + "root = str(Path(\".\"))\n", "work_dir = os.path.join(root, 'ref_api_work_dir')\n", "if not os.path.isdir(work_dir):\n", " os.makedirs(work_dir)\n", "\n", - "msd_task = \"Task09_Spleen\"\n", + "msd_task = \"Task04_Hippocampus\"\n", "dataroot = os.path.join(root, msd_task)\n", "datalist_file = os.path.join(\"..\", \"tasks\", \"msd\", msd_task, \"msd_\" + msd_task.lower() + \"_folds.json\")\n", "\n", @@ -114,7 +109,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -157,21 +152,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "100%|██████████| 40/40 [01:34<00:00, 2.37s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2022-09-29 01:43:36,891 - WARNING - Data is not completely uniform. MONAI transforms may provide unexpected result\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n" + "100%|██████████| 30/30 [00:00<00:00, 71.04it/s]\n" ] } ], @@ -188,7 +169,7 @@ "Besides the Python API call, user can also use command line interface (CLI) provided by the user's OS. One example is the following bash commands:\n", "\n", "```bash\n", - "python -m monai.apps.auto3dseg DataAnalyzer get_all_case_stats --datalist=\"../tasks/msd/Task09_Spleen/msd_task09_spleen_folds.json\" --dataroot=\"./Task09_Spleen\" --output_path=\"./ref_api_work_dir/data_stats.yaml\"\n", + "python -m monai.apps.auto3dseg DataAnalyzer get_all_case_stats --datalist=\"../tasks/msd/Task04_Hippocampus/msd_task04_hippocampus_folds.json\" --dataroot=\"./Task04_Hippocampus\" --output_path=\"./ref_api_work_dir/data_stats.yaml\"\n", "```" ] }, @@ -214,16 +195,16 @@ "name": "stderr", "output_type": "stream", "text": [ - "algo_templates.tar.gz: 100%|██████████| 280k/280k [00:00<00:00, 607kB/s] " + "algo_templates.tar.gz: 296kB [00:01, 247kB/s] " ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2022-09-29 01:43:37,829 - INFO - Downloaded: /tmp/tmpodiia3qq/algo_templates.tar.gz\n", - "2022-09-29 01:43:37,830 - INFO - Expected md5 is None, skip md5 check for file /tmp/tmpodiia3qq/algo_templates.tar.gz.\n", - "2022-09-29 01:43:37,830 - INFO - Writing into directory: ./ref_api_work_dir.\n" + "2022-10-18 08:14:35,703 - INFO - Downloaded: /tmp/tmps2pt5u7y/algo_templates.tar.gz\n", + "2022-10-18 08:14:35,705 - INFO - Expected md5 is None, skip md5 check for file /tmp/tmps2pt5u7y/algo_templates.tar.gz.\n", + "2022-10-18 08:14:35,706 - INFO - Writing into directory: ./ref_api_work_dir.\n" ] }, { @@ -237,26 +218,26 @@ "name": "stdout", "output_type": "stream", "text": [ - "2022-09-29 01:43:38,651 - INFO - ./ref_api_work_dir/segresnet2d_0\n", - "2022-09-29 01:43:39,029 - INFO - ./ref_api_work_dir/segresnet2d_1\n", - "2022-09-29 01:43:39,411 - INFO - ./ref_api_work_dir/segresnet2d_2\n", - "2022-09-29 01:43:39,916 - INFO - ./ref_api_work_dir/segresnet2d_3\n", - "2022-09-29 01:43:40,373 - INFO - ./ref_api_work_dir/segresnet2d_4\n", - "2022-09-29 01:43:41,656 - INFO - ./ref_api_work_dir/dints_0\n", - "2022-09-29 01:43:42,284 - INFO - ./ref_api_work_dir/dints_1\n", - "2022-09-29 01:43:42,678 - INFO - ./ref_api_work_dir/dints_2\n", - "2022-09-29 01:43:43,172 - INFO - ./ref_api_work_dir/dints_3\n", - "2022-09-29 01:43:43,681 - INFO - ./ref_api_work_dir/dints_4\n", - "2022-09-29 01:43:44,448 - INFO - ./ref_api_work_dir/swinunetr_0\n", - "2022-09-29 01:43:44,807 - INFO - ./ref_api_work_dir/swinunetr_1\n", - "2022-09-29 01:43:45,177 - INFO - ./ref_api_work_dir/swinunetr_2\n", - "2022-09-29 01:43:45,687 - INFO - ./ref_api_work_dir/swinunetr_3\n", - "2022-09-29 01:43:48,313 - INFO - ./ref_api_work_dir/swinunetr_4\n", - "2022-09-29 01:43:48,983 - INFO - ./ref_api_work_dir/segresnet_0\n", - "2022-09-29 01:43:49,597 - INFO - ./ref_api_work_dir/segresnet_1\n", - "2022-09-29 01:43:50,008 - INFO - ./ref_api_work_dir/segresnet_2\n", - "2022-09-29 01:43:50,393 - INFO - ./ref_api_work_dir/segresnet_3\n", - "2022-09-29 01:43:51,990 - INFO - ./ref_api_work_dir/segresnet_4\n" + "2022-10-18 08:14:36,019 - INFO - ./ref_api_work_dir/segresnet2d_0\n", + "2022-10-18 08:14:36,270 - INFO - ./ref_api_work_dir/segresnet2d_1\n", + "2022-10-18 08:14:36,616 - INFO - ./ref_api_work_dir/segresnet2d_2\n", + "2022-10-18 08:14:36,879 - INFO - ./ref_api_work_dir/segresnet2d_3\n", + "2022-10-18 08:14:37,142 - INFO - ./ref_api_work_dir/segresnet2d_4\n", + "2022-10-18 08:14:37,412 - INFO - ./ref_api_work_dir/dints_0\n", + "2022-10-18 08:14:37,799 - INFO - ./ref_api_work_dir/dints_1\n", + "2022-10-18 08:14:38,071 - INFO - ./ref_api_work_dir/dints_2\n", + "2022-10-18 08:14:38,343 - INFO - ./ref_api_work_dir/dints_3\n", + "2022-10-18 08:14:38,717 - INFO - ./ref_api_work_dir/dints_4\n", + "2022-10-18 08:14:38,975 - INFO - ./ref_api_work_dir/swinunetr_0\n", + "2022-10-18 08:14:39,233 - INFO - ./ref_api_work_dir/swinunetr_1\n", + "2022-10-18 08:14:39,492 - INFO - ./ref_api_work_dir/swinunetr_2\n", + "2022-10-18 08:14:39,849 - INFO - ./ref_api_work_dir/swinunetr_3\n", + "2022-10-18 08:14:40,106 - INFO - ./ref_api_work_dir/swinunetr_4\n", + "2022-10-18 08:14:40,398 - INFO - ./ref_api_work_dir/segresnet_0\n", + "2022-10-18 08:14:40,771 - INFO - ./ref_api_work_dir/segresnet_1\n", + "2022-10-18 08:14:41,042 - INFO - ./ref_api_work_dir/segresnet_2\n", + "2022-10-18 08:14:41,309 - INFO - ./ref_api_work_dir/segresnet_3\n", + "2022-10-18 08:14:41,574 - INFO - ./ref_api_work_dir/segresnet_4\n" ] } ], @@ -318,17 +299,9 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'num_iterations': 32000, 'num_iterations_per_validation': 16000, 'num_images_per_batch': 2, 'num_epochs': 2000, 'num_warmup_iterations': 16000}\n" - ] - } - ], + "outputs": [], "source": [ "# The training params are optional. If you don't want to change the default settings, you can use either `train()` or `train({})` in 2.3.2 \n", "\n", @@ -370,7 +343,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -438,7 +411,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -466,7 +439,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.12" + "version": "3.8.13" }, "vscode": { "interpreter": { diff --git a/auto3dseg/notebooks/auto3dseg_hello_world.ipynb b/auto3dseg/notebooks/auto3dseg_hello_world.ipynb index 03030e7d7c..198cf7e75d 100644 --- a/auto3dseg/notebooks/auto3dseg_hello_world.ipynb +++ b/auto3dseg/notebooks/auto3dseg_hello_world.ipynb @@ -55,6 +55,8 @@ "import numpy as np\n", "import matplotlib.pyplot as plt\n", "\n", + "from pathlib import Path\n", + "\n", "from monai.apps.auto3dseg import AutoRunner\n", "from monai.data import create_test_image_3d" ] @@ -115,7 +117,7 @@ "metadata": {}, "outputs": [], "source": [ - "work_dir = \"./helloworld_work_dir\"\n", + "work_dir = str(Path(\"./helloworld_work_dir\"))\n", "if not os.path.isdir(work_dir):\n", " os.makedirs(work_dir)\n", "\n", @@ -199,9 +201,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "2022-09-22 03:35:21,886 - INFO - Work directory ./auto3dseg_work_dir is used to save all results\n", - "2022-09-22 03:35:21,895 - INFO - The output_dir is not specified. /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/ensemble_output will be used to save ensemble predictions\n", - "2022-09-22 03:35:21,897 - INFO - Directory /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/ensemble_output is created to save ensemble predictions\n" + "2022-10-18 08:27:25,835 - INFO - Work directory helloworld_work_dir is used to save all results\n", + "2022-10-18 08:27:25,841 - INFO - The output_dir is not specified. /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/ensemble_output will be used to save ensemble predictions\n", + "2022-10-18 08:27:25,841 - INFO - Directory /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/ensemble_output is created to save ensemble predictions\n" ] } ], @@ -263,17 +265,17 @@ "name": "stderr", "output_type": "stream", "text": [ - "100%|██████████| 12/12 [00:02<00:00, 4.84it/s]\n", - "algo_templates.tar.gz: 100%|██████████| 280k/280k [00:00<00:00, 382kB/s] " + "100%|██████████| 12/12 [00:00<00:00, 32.40it/s]\n", + "algo_templates.tar.gz: 296kB [00:01, 257kB/s] " ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2022-09-22 03:35:29,243 - INFO - Downloaded: /tmp/tmpvkd67w4k/algo_templates.tar.gz\n", - "2022-09-22 03:35:29,245 - INFO - Expected md5 is None, skip md5 check for file /tmp/tmpvkd67w4k/algo_templates.tar.gz.\n", - "2022-09-22 03:35:29,247 - INFO - Writing into directory: /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir.\n" + "2022-10-18 08:27:27,916 - INFO - Downloaded: /tmp/tmpkajbqsnz/algo_templates.tar.gz\n", + "2022-10-18 08:27:27,918 - INFO - Expected md5 is None, skip md5 check for file /tmp/tmpkajbqsnz/algo_templates.tar.gz.\n", + "2022-10-18 08:27:27,920 - INFO - Writing into directory: /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir.\n" ] }, { @@ -287,12 +289,12 @@ "name": "stdout", "output_type": "stream", "text": [ - "2022-09-22 03:35:29,440 - INFO - /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/segresnet2d_0\n", - "2022-09-22 03:35:29,646 - INFO - /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/dints_0\n", - "2022-09-22 03:35:29,761 - INFO - /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/swinunetr_0\n", - "2022-09-22 03:35:29,885 - INFO - /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/segresnet_0\n", - "2022-09-22 03:35:29,886 - INFO - Launching: python /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/segresnet_0/scripts/train.py run --config_file='/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/segresnet_0/configs/transforms_validate.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/segresnet_0/configs/transforms_train.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/segresnet_0/configs/transforms_infer.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/segresnet_0/configs/network.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/segresnet_0/configs/hyper_parameters.yaml' --num_iterations=8 --num_iterations_per_validation=4 --num_images_per_batch=2 --num_epochs=2 --num_warmup_iterations=4\n", - "2022-09-22 03:35:35,964 - INFO - CompletedProcess(args=['python', '/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/segresnet_0/scripts/train.py', 'run', \"--config_file='/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/segresnet_0/configs/transforms_validate.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/segresnet_0/configs/transforms_train.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/segresnet_0/configs/transforms_infer.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/segresnet_0/configs/network.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/segresnet_0/configs/hyper_parameters.yaml'\", '--num_iterations=8', '--num_iterations_per_validation=4', '--num_images_per_batch=2', '--num_epochs=2', '--num_warmup_iterations=4'], returncode=0, stdout=b\"[info] number of GPUs: 1\n", + "2022-10-18 08:27:28,111 - INFO - /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/segresnet2d_0\n", + "2022-10-18 08:27:28,237 - INFO - /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/dints_0\n", + "2022-10-18 08:27:28,351 - INFO - /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/swinunetr_0\n", + "2022-10-18 08:27:28,509 - INFO - /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/segresnet_0\n", + "2022-10-18 08:27:28,510 - INFO - Launching: python /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/segresnet_0/scripts/train.py run --config_file='/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/segresnet_0/configs/transforms_validate.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/segresnet_0/configs/transforms_train.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/segresnet_0/configs/transforms_infer.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/segresnet_0/configs/network.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/segresnet_0/configs/hyper_parameters.yaml' --num_iterations=8 --num_iterations_per_validation=4 --num_images_per_batch=2 --num_epochs=2 --num_warmup_iterations=4\n", + "2022-10-18 08:27:35,294 - INFO - CompletedProcess(args=['python', '/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/segresnet_0/scripts/train.py', 'run', \"--config_file='/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/segresnet_0/configs/transforms_validate.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/segresnet_0/configs/transforms_train.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/segresnet_0/configs/transforms_infer.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/segresnet_0/configs/network.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/segresnet_0/configs/hyper_parameters.yaml'\", '--num_iterations=8', '--num_iterations_per_validation=4', '--num_images_per_batch=2', '--num_epochs=2', '--num_warmup_iterations=4'], returncode=0, stdout=b\"[info] number of GPUs: 1\n", "[info] world_size: 1\n", "train_files: 8\n", "val_files: 4\n", @@ -303,40 +305,40 @@ "----------\n", "epoch 1/2\n", "learning rate is set to 2e-05\n", - "[2022-09-22 03:35:33] 1/4, train_loss: 1.2582\n", - "[2022-09-22 03:35:33] 2/4, train_loss: 1.2323\n", - "[2022-09-22 03:35:33] 3/4, train_loss: 1.0489\n", - "[2022-09-22 03:35:33] 4/4, train_loss: 1.0692\n", - "epoch 1 average loss: 1.1521, best mean dice: -1.0000 at epoch -1\n", - "1 / 4 tensor([[0.6500]], device='cuda:0')\n", - "2 / 4 tensor([[0.6500]], device='cuda:0')\n", - "3 / 4 tensor([[0.6500]], device='cuda:0')\n", - "4 / 4 tensor([[0.6500]], device='cuda:0')\n", - "evaluation metric - class 1: 0.6500166058540344\n", - "avg_metric 0.6500166058540344\n", + "[2022-10-18 08:27:32] 1/4, train_loss: 1.4624\n", + "[2022-10-18 08:27:33] 2/4, train_loss: 1.4597\n", + "[2022-10-18 08:27:33] 3/4, train_loss: 1.3316\n", + "[2022-10-18 08:27:33] 4/4, train_loss: 1.3588\n", + "epoch 1 average loss: 1.4031, best mean dice: -1.0000 at epoch -1\n", + "1 / 4 tensor([[0.1104]], device='cuda:0')\n", + "2 / 4 tensor([[0.1104]], device='cuda:0')\n", + "3 / 4 tensor([[0.1104]], device='cuda:0')\n", + "4 / 4 tensor([[0.1104]], device='cuda:0')\n", + "evaluation metric - class 1: 0.11038827896118164\n", + "avg_metric 0.11038827896118164\n", "saved new best metric model\n", - "current epoch: 1 current mean dice: 0.6500 best mean dice: 0.6500 at epoch 1\n", + "current epoch: 1 current mean dice: 0.1104 best mean dice: 0.1104 at epoch 1\n", "----------\n", "epoch 2/2\n", "learning rate is set to 0.0\n", - "[2022-09-22 03:35:34] 1/4, train_loss: 0.9847\n", - "[2022-09-22 03:35:34] 2/4, train_loss: 0.9677\n", - "[2022-09-22 03:35:34] 3/4, train_loss: 0.9508\n", - "[2022-09-22 03:35:34] 4/4, train_loss: 0.9461\n", - "epoch 2 average loss: 0.9623, best mean dice: 0.6500 at epoch 1\n", - "1 / 4 tensor([[0.6691]], device='cuda:0')\n", - "2 / 4 tensor([[0.6691]], device='cuda:0')\n", - "3 / 4 tensor([[0.6691]], device='cuda:0')\n", - "4 / 4 tensor([[0.6691]], device='cuda:0')\n", - "evaluation metric - class 1: 0.6691252589225769\n", - "avg_metric 0.6691252589225769\n", + "[2022-10-18 08:27:34] 1/4, train_loss: 1.3096\n", + "[2022-10-18 08:27:34] 2/4, train_loss: 1.3398\n", + "[2022-10-18 08:27:34] 3/4, train_loss: 1.2666\n", + "[2022-10-18 08:27:34] 4/4, train_loss: 1.2679\n", + "epoch 2 average loss: 1.2960, best mean dice: 0.1104 at epoch 1\n", + "1 / 4 tensor([[0.1236]], device='cuda:0')\n", + "2 / 4 tensor([[0.1236]], device='cuda:0')\n", + "3 / 4 tensor([[0.1236]], device='cuda:0')\n", + "4 / 4 tensor([[0.1236]], device='cuda:0')\n", + "evaluation metric - class 1: 0.12358943372964859\n", + "avg_metric 0.12358943372964859\n", "saved new best metric model\n", - "current epoch: 2 current mean dice: 0.6691 best mean dice: 0.6691 at epoch 2\n", - "train completed, best_metric: 0.6691 at epoch: 2\n", - "0.6691252589225769\n", + "current epoch: 2 current mean dice: 0.1236 best mean dice: 0.1236 at epoch 2\n", + "train completed, best_metric: 0.1236 at epoch: 2\n", + "0.12358943372964859\n", "\", stderr=b'')\n", - "2022-09-22 03:35:35,972 - INFO - Launching: python /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/swinunetr_0/scripts/train.py run --config_file='/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/swinunetr_0/configs/transforms_validate.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/swinunetr_0/configs/transforms_train.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/swinunetr_0/configs/transforms_infer.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/swinunetr_0/configs/network.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/swinunetr_0/configs/hyper_parameters.yaml' --num_iterations=8 --num_iterations_per_validation=4 --num_images_per_batch=2 --num_epochs=2 --num_warmup_iterations=4\n", - "2022-09-22 03:35:42,971 - INFO - CompletedProcess(args=['python', '/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/swinunetr_0/scripts/train.py', 'run', \"--config_file='/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/swinunetr_0/configs/transforms_validate.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/swinunetr_0/configs/transforms_train.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/swinunetr_0/configs/transforms_infer.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/swinunetr_0/configs/network.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/swinunetr_0/configs/hyper_parameters.yaml'\", '--num_iterations=8', '--num_iterations_per_validation=4', '--num_images_per_batch=2', '--num_epochs=2', '--num_warmup_iterations=4'], returncode=0, stdout=b\"[info] number of GPUs: 1\n", + "2022-10-18 08:27:35,301 - INFO - Launching: python /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/swinunetr_0/scripts/train.py run --config_file='/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/swinunetr_0/configs/transforms_validate.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/swinunetr_0/configs/transforms_train.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/swinunetr_0/configs/transforms_infer.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/swinunetr_0/configs/network.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/swinunetr_0/configs/hyper_parameters.yaml' --num_iterations=8 --num_iterations_per_validation=4 --num_images_per_batch=2 --num_epochs=2 --num_warmup_iterations=4\n", + "2022-10-18 08:27:42,721 - INFO - CompletedProcess(args=['python', '/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/swinunetr_0/scripts/train.py', 'run', \"--config_file='/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/swinunetr_0/configs/transforms_validate.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/swinunetr_0/configs/transforms_train.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/swinunetr_0/configs/transforms_infer.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/swinunetr_0/configs/network.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/swinunetr_0/configs/hyper_parameters.yaml'\", '--num_iterations=8', '--num_iterations_per_validation=4', '--num_images_per_batch=2', '--num_epochs=2', '--num_warmup_iterations=4'], returncode=0, stdout=b\"[info] number of GPUs: 1\n", "[info] world_size: 1\n", "train_files: 8\n", "val_files: 4\n", @@ -347,40 +349,40 @@ "----------\n", "epoch 1/2\n", "learning rate is set to 0.0001\n", - "[2022-09-22 03:35:39] 1/4, train_loss: 1.1162\n", - "[2022-09-22 03:35:40] 2/4, train_loss: 0.9896\n", - "[2022-09-22 03:35:40] 3/4, train_loss: 0.9468\n", - "[2022-09-22 03:35:40] 4/4, train_loss: 0.9314\n", - "epoch 1 average loss: 0.9960, best mean dice: -1.0000 at epoch -1\n", - "1 / 4 tensor([[0.6473]], device='cuda:0')\n", - "2 / 4 tensor([[0.6473]], device='cuda:0')\n", - "3 / 4 tensor([[0.6473]], device='cuda:0')\n", - "4 / 4 tensor([[0.6473]], device='cuda:0')\n", - "evaluation metric - class 1: 0.6472799777984619\n", - "avg_metric 0.6472799777984619\n", + "[2022-10-18 08:27:39] 1/4, train_loss: 1.1682\n", + "[2022-10-18 08:27:39] 2/4, train_loss: 1.0048\n", + "[2022-10-18 08:27:39] 3/4, train_loss: 0.9436\n", + "[2022-10-18 08:27:40] 4/4, train_loss: 0.9151\n", + "epoch 1 average loss: 1.0079, best mean dice: -1.0000 at epoch -1\n", + "1 / 4 tensor([[0.6443]], device='cuda:0')\n", + "2 / 4 tensor([[0.6443]], device='cuda:0')\n", + "3 / 4 tensor([[0.6443]], device='cuda:0')\n", + "4 / 4 tensor([[0.6443]], device='cuda:0')\n", + "evaluation metric - class 1: 0.6442577242851257\n", + "avg_metric 0.6442577242851257\n", "saved new best metric model\n", - "current epoch: 1 current mean dice: 0.6473 best mean dice: 0.6473 at epoch 1\n", + "current epoch: 1 current mean dice: 0.6443 best mean dice: 0.6443 at epoch 1\n", "----------\n", "epoch 2/2\n", "learning rate is set to 6.25e-06\n", - "[2022-09-22 03:35:41] 1/4, train_loss: 0.9178\n", - "[2022-09-22 03:35:41] 2/4, train_loss: 0.9157\n", - "[2022-09-22 03:35:41] 3/4, train_loss: 0.9099\n", - "[2022-09-22 03:35:41] 4/4, train_loss: 0.9090\n", - "epoch 2 average loss: 0.9131, best mean dice: 0.6473 at epoch 1\n", - "1 / 4 tensor([[0.6537]], device='cuda:0')\n", - "2 / 4 tensor([[0.6537]], device='cuda:0')\n", - "3 / 4 tensor([[0.6537]], device='cuda:0')\n", - "4 / 4 tensor([[0.6537]], device='cuda:0')\n", - "evaluation metric - class 1: 0.6537249088287354\n", - "avg_metric 0.6537249088287354\n", + "[2022-10-18 08:27:41] 1/4, train_loss: 0.9040\n", + "[2022-10-18 08:27:41] 2/4, train_loss: 0.8981\n", + "[2022-10-18 08:27:41] 3/4, train_loss: 0.8954\n", + "[2022-10-18 08:27:41] 4/4, train_loss: 0.8919\n", + "epoch 2 average loss: 0.8974, best mean dice: 0.6443 at epoch 1\n", + "1 / 4 tensor([[0.6603]], device='cuda:0')\n", + "2 / 4 tensor([[0.6603]], device='cuda:0')\n", + "3 / 4 tensor([[0.6603]], device='cuda:0')\n", + "4 / 4 tensor([[0.6603]], device='cuda:0')\n", + "evaluation metric - class 1: 0.6603014469146729\n", + "avg_metric 0.6603014469146729\n", "saved new best metric model\n", - "current epoch: 2 current mean dice: 0.6537 best mean dice: 0.6537 at epoch 2\n", - "train completed, best_metric: 0.6537 at epoch: 2\n", - "0.6537249088287354\n", + "current epoch: 2 current mean dice: 0.6603 best mean dice: 0.6603 at epoch 2\n", + "train completed, best_metric: 0.6603 at epoch: 2\n", + "0.6603014469146729\n", "\", stderr=b'')\n", - "2022-09-22 03:35:42,978 - INFO - Launching: python /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/dints_0/scripts/search.py run --config_file='/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/dints_0/configs/transforms_validate.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/dints_0/configs/transforms_train.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/dints_0/configs/transforms_infer.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/dints_0/configs/hyper_parameters_search.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/dints_0/configs/network.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/dints_0/configs/network_search.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/dints_0/configs/hyper_parameters.yaml' --searching#num_iterations=8 --searching#num_iterations_per_validation=4 --searching#num_images_per_batch=2 --searching#num_epochs=2 --searching#num_warmup_iterations=4\n", - "2022-09-22 03:36:10,957 - INFO - CompletedProcess(args=['python', '/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/dints_0/scripts/search.py', 'run', \"--config_file='/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/dints_0/configs/transforms_validate.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/dints_0/configs/transforms_train.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/dints_0/configs/transforms_infer.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/dints_0/configs/hyper_parameters_search.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/dints_0/configs/network.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/dints_0/configs/network_search.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/dints_0/configs/hyper_parameters.yaml'\", '--searching#num_iterations=8', '--searching#num_iterations_per_validation=4', '--searching#num_images_per_batch=2', '--searching#num_epochs=2', '--searching#num_warmup_iterations=4'], returncode=0, stdout=b\"[info] number of GPUs: 1\n", + "2022-10-18 08:27:42,728 - INFO - Launching: python /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/dints_0/scripts/search.py run --config_file='/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/dints_0/configs/transforms_validate.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/dints_0/configs/transforms_train.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/dints_0/configs/transforms_infer.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/dints_0/configs/hyper_parameters_search.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/dints_0/configs/network.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/dints_0/configs/network_search.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/dints_0/configs/hyper_parameters.yaml' --searching#num_iterations=8 --searching#num_iterations_per_validation=4 --searching#num_images_per_batch=2 --searching#num_epochs=2 --searching#num_warmup_iterations=4\n", + "2022-10-18 08:28:11,311 - INFO - CompletedProcess(args=['python', '/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/dints_0/scripts/search.py', 'run', \"--config_file='/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/dints_0/configs/transforms_validate.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/dints_0/configs/transforms_train.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/dints_0/configs/transforms_infer.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/dints_0/configs/hyper_parameters_search.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/dints_0/configs/network.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/dints_0/configs/network_search.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/dints_0/configs/hyper_parameters.yaml'\", '--searching#num_iterations=8', '--searching#num_iterations_per_validation=4', '--searching#num_images_per_batch=2', '--searching#num_epochs=2', '--searching#num_warmup_iterations=4'], returncode=0, stdout=b\"[info] number of GPUs: 1\n", "[info] world_size: 1\n", "train_files_w: 4\n", "train_files_a: 4\n", @@ -392,53 +394,53 @@ "----------\n", "epoch 1/4\n", "learning rate is set to 0.025\n", - "[2022-09-22 03:35:48] 1/2, train_loss: 0.8420\n", - "[2022-09-22 03:35:49] 2/2, train_loss: 0.8201\n", - "epoch 1 average loss: 0.8310, best mean dice: -1.0000 at epoch -1\n", + "[2022-10-18 08:27:48] 1/2, train_loss: 0.8553\n", + "[2022-10-18 08:27:49] 2/2, train_loss: 0.8300\n", + "epoch 1 average loss: 0.8426, best mean dice: -1.0000 at epoch -1\n", "----------\n", "epoch 2/4\n", "learning rate is set to 0.025\n", - "[2022-09-22 03:35:50] 1/2, train_loss: 0.8137\n", - "[2022-09-22 03:35:50] 2/2, train_loss: 0.8127\n", - "epoch 2 average loss: 0.8132, best mean dice: -1.0000 at epoch -1\n", - "1 / 4 tensor([[0.1347]], device='cuda:0')\n", - "2 / 4 tensor([[0.1347]], device='cuda:0')\n", - "3 / 4 tensor([[0.1347]], device='cuda:0')\n", - "4 / 4 tensor([[0.1347]], device='cuda:0')\n", - "evaluation metric - class 1: 0.13466915488243103\n", - "avg_metric 0.13466915488243103\n", + "[2022-10-18 08:27:50] 1/2, train_loss: 0.8394\n", + "[2022-10-18 08:27:50] 2/2, train_loss: 0.8302\n", + "epoch 2 average loss: 0.8348, best mean dice: -1.0000 at epoch -1\n", + "1 / 4 tensor([[0.1750]], device='cuda:0')\n", + "2 / 4 tensor([[0.1750]], device='cuda:0')\n", + "3 / 4 tensor([[0.1750]], device='cuda:0')\n", + "4 / 4 tensor([[0.1750]], device='cuda:0')\n", + "evaluation metric - class 1: 0.17498305439949036\n", + "avg_metric 0.17498305439949036\n", "saved new best metric model\n", - "current epoch: 2 current mean dice: 0.1347 best mean dice: 0.1347 at epoch 2\n", + "current epoch: 2 current mean dice: 0.1750 best mean dice: 0.1750 at epoch 2\n", "----------\n", "epoch 3/4\n", "learning rate is set to 0.025\n", - "[2022-09-22 03:35:58] 1/2, train_loss: 0.7776\n", - "[2022-09-22 03:35:59] 1/2, train_loss_arch: 0.7652\n", - "[2022-09-22 03:36:00] 2/2, train_loss: 0.7906\n", - "[2022-09-22 03:36:00] 2/2, train_loss_arch: 0.7980\n", - "epoch 3 average loss: 0.7841, best mean dice: 0.1347 at epoch 2\n", - "epoch 3 average arch loss: 0.7816, best mean dice: 0.1347 at epoch 2\n", + "[2022-10-18 08:27:58] 1/2, train_loss: 0.8238\n", + "[2022-10-18 08:27:59] 1/2, train_loss_arch: 0.8115\n", + "[2022-10-18 08:28:00] 2/2, train_loss: 0.8093\n", + "[2022-10-18 08:28:00] 2/2, train_loss_arch: 0.8043\n", + "epoch 3 average loss: 0.8166, best mean dice: 0.1750 at epoch 2\n", + "epoch 3 average arch loss: 0.8079, best mean dice: 0.1750 at epoch 2\n", "----------\n", "epoch 4/4\n", "learning rate is set to 0.00625\n", - "[2022-09-22 03:36:01] 1/2, train_loss: 0.7761\n", - "[2022-09-22 03:36:02] 1/2, train_loss_arch: 1.1109\n", - "[2022-09-22 03:36:03] 2/2, train_loss: 0.7981\n", - "[2022-09-22 03:36:03] 2/2, train_loss_arch: 1.0805\n", - "epoch 4 average loss: 0.7871, best mean dice: 0.1347 at epoch 2\n", - "epoch 4 average arch loss: 1.0957, best mean dice: 0.1347 at epoch 2\n", - "1 / 4 tensor([[0.1877]], device='cuda:0')\n", - "2 / 4 tensor([[0.1877]], device='cuda:0')\n", - "3 / 4 tensor([[0.1877]], device='cuda:0')\n", - "4 / 4 tensor([[0.1877]], device='cuda:0')\n", - "evaluation metric - class 1: 0.18768733739852905\n", - "avg_metric 0.18768733739852905\n", + "[2022-10-18 08:28:01] 1/2, train_loss: 0.7779\n", + "[2022-10-18 08:28:02] 1/2, train_loss_arch: 1.1365\n", + "[2022-10-18 08:28:03] 2/2, train_loss: 0.8037\n", + "[2022-10-18 08:28:04] 2/2, train_loss_arch: 1.1294\n", + "epoch 4 average loss: 0.7908, best mean dice: 0.1750 at epoch 2\n", + "epoch 4 average arch loss: 1.1330, best mean dice: 0.1750 at epoch 2\n", + "1 / 4 tensor([[0.1844]], device='cuda:0')\n", + "2 / 4 tensor([[0.1844]], device='cuda:0')\n", + "3 / 4 tensor([[0.1844]], device='cuda:0')\n", + "4 / 4 tensor([[0.1844]], device='cuda:0')\n", + "evaluation metric - class 1: 0.1843988001346588\n", + "avg_metric 0.1843988001346588\n", "saved new best metric model\n", - "current epoch: 4 current mean dice: 0.1877 best mean dice: 0.1877 at epoch 4\n", - "train completed, best_metric: 0.1877 at epoch: 4\n", + "current epoch: 4 current mean dice: 0.1844 best mean dice: 0.1844 at epoch 4\n", + "train completed, best_metric: 0.1844 at epoch: 4\n", "\", stderr=b'')\n", - "2022-09-22 03:36:10,958 - INFO - Launching: python /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/dints_0/scripts/train.py run --config_file='/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/dints_0/configs/transforms_validate.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/dints_0/configs/transforms_train.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/dints_0/configs/transforms_infer.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/dints_0/configs/hyper_parameters_search.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/dints_0/configs/network.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/dints_0/configs/network_search.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/dints_0/configs/hyper_parameters.yaml' --training#num_iterations=8 --training#num_iterations_per_validation=4 --training#num_images_per_batch=2 --training#num_epochs=2 --training#num_warmup_iterations=4\n", - "2022-09-22 03:36:21,254 - INFO - CompletedProcess(args=['python', '/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/dints_0/scripts/train.py', 'run', \"--config_file='/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/dints_0/configs/transforms_validate.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/dints_0/configs/transforms_train.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/dints_0/configs/transforms_infer.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/dints_0/configs/hyper_parameters_search.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/dints_0/configs/network.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/dints_0/configs/network_search.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/dints_0/configs/hyper_parameters.yaml'\", '--training#num_iterations=8', '--training#num_iterations_per_validation=4', '--training#num_images_per_batch=2', '--training#num_epochs=2', '--training#num_warmup_iterations=4'], returncode=0, stdout=b\"[info] number of GPUs: 1\n", + "2022-10-18 08:28:11,312 - INFO - Launching: python /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/dints_0/scripts/train.py run --config_file='/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/dints_0/configs/transforms_validate.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/dints_0/configs/transforms_train.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/dints_0/configs/transforms_infer.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/dints_0/configs/hyper_parameters_search.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/dints_0/configs/network.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/dints_0/configs/network_search.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/dints_0/configs/hyper_parameters.yaml' --training#num_iterations=8 --training#num_iterations_per_validation=4 --training#num_images_per_batch=2 --training#num_epochs=2 --training#num_warmup_iterations=4\n", + "2022-10-18 08:28:21,854 - INFO - CompletedProcess(args=['python', '/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/dints_0/scripts/train.py', 'run', \"--config_file='/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/dints_0/configs/transforms_validate.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/dints_0/configs/transforms_train.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/dints_0/configs/transforms_infer.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/dints_0/configs/hyper_parameters_search.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/dints_0/configs/network.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/dints_0/configs/network_search.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/dints_0/configs/hyper_parameters.yaml'\", '--training#num_iterations=8', '--training#num_iterations_per_validation=4', '--training#num_images_per_batch=2', '--training#num_epochs=2', '--training#num_warmup_iterations=4'], returncode=0, stdout=b\"[info] number of GPUs: 1\n", "[info] world_size: 1\n", "train_files: 8\n", "val_files: 4\n", @@ -449,40 +451,40 @@ "----------\n", "epoch 1/2\n", "learning rate is set to 0.2\n", - "[2022-09-22 03:36:15] 1/4, train_loss: 0.7647\n", - "[2022-09-22 03:36:16] 2/4, train_loss: 0.6694\n", - "[2022-09-22 03:36:16] 3/4, train_loss: 0.6844\n", - "[2022-09-22 03:36:16] 4/4, train_loss: 0.6755\n", - "epoch 1 average loss: 0.6985, best mean dice: -1.0000 at epoch -1\n", - "1 / 4 tensor([[0.3745]], device='cuda:0')\n", - "2 / 4 tensor([[0.3745]], device='cuda:0')\n", - "3 / 4 tensor([[0.3745]], device='cuda:0')\n", - "4 / 4 tensor([[0.3745]], device='cuda:0')\n", - "evaluation metric - class 1: 0.3745383620262146\n", - "avg_metric 0.3745383620262146\n", + "[2022-10-18 08:28:16] 1/4, train_loss: 0.7010\n", + "[2022-10-18 08:28:17] 2/4, train_loss: 0.6845\n", + "[2022-10-18 08:28:17] 3/4, train_loss: 0.6449\n", + "[2022-10-18 08:28:17] 4/4, train_loss: 0.6553\n", + "epoch 1 average loss: 0.6714, best mean dice: -1.0000 at epoch -1\n", + "1 / 4 tensor([[0.2394]], device='cuda:0')\n", + "2 / 4 tensor([[0.2394]], device='cuda:0')\n", + "3 / 4 tensor([[0.2394]], device='cuda:0')\n", + "4 / 4 tensor([[0.2394]], device='cuda:0')\n", + "evaluation metric - class 1: 0.2393515557050705\n", + "avg_metric 0.2393515557050705\n", "saved new best metric model\n", - "current epoch: 1 current mean dice: 0.3745 best mean dice: 0.3745 at epoch 1\n", + "current epoch: 1 current mean dice: 0.2394 best mean dice: 0.2394 at epoch 1\n", "----------\n", "epoch 2/2\n", "learning rate is set to 0.0125\n", - "[2022-09-22 03:36:18] 1/4, train_loss: 0.6489\n", - "[2022-09-22 03:36:18] 2/4, train_loss: 0.6701\n", - "[2022-09-22 03:36:19] 3/4, train_loss: 0.6609\n", - "[2022-09-22 03:36:19] 4/4, train_loss: 0.6555\n", - "epoch 2 average loss: 0.6588, best mean dice: 0.3745 at epoch 1\n", - "1 / 4 tensor([[0.3788]], device='cuda:0')\n", - "2 / 4 tensor([[0.3788]], device='cuda:0')\n", - "3 / 4 tensor([[0.3788]], device='cuda:0')\n", - "4 / 4 tensor([[0.3788]], device='cuda:0')\n", - "evaluation metric - class 1: 0.37883225083351135\n", - "avg_metric 0.37883225083351135\n", + "[2022-10-18 08:28:19] 1/4, train_loss: 0.6216\n", + "[2022-10-18 08:28:19] 2/4, train_loss: 0.5904\n", + "[2022-10-18 08:28:19] 3/4, train_loss: 0.6117\n", + "[2022-10-18 08:28:20] 4/4, train_loss: 0.6234\n", + "epoch 2 average loss: 0.6118, best mean dice: 0.2394 at epoch 1\n", + "1 / 4 tensor([[0.2492]], device='cuda:0')\n", + "2 / 4 tensor([[0.2492]], device='cuda:0')\n", + "3 / 4 tensor([[0.2492]], device='cuda:0')\n", + "4 / 4 tensor([[0.2492]], device='cuda:0')\n", + "evaluation metric - class 1: 0.24918213486671448\n", + "avg_metric 0.24918213486671448\n", "saved new best metric model\n", - "current epoch: 2 current mean dice: 0.3788 best mean dice: 0.3788 at epoch 2\n", - "train completed, best_metric: 0.3788 at epoch: 2\n", - "0.37883225083351135\n", + "current epoch: 2 current mean dice: 0.2492 best mean dice: 0.2492 at epoch 2\n", + "train completed, best_metric: 0.2492 at epoch: 2\n", + "0.24918213486671448\n", "\", stderr=b'')\n", - "2022-09-22 03:36:21,261 - INFO - Launching: python /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/segresnet2d_0/scripts/train.py run --config_file='/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/segresnet2d_0/configs/transforms_validate.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/segresnet2d_0/configs/transforms_train.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/segresnet2d_0/configs/transforms_infer.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/segresnet2d_0/configs/network.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/segresnet2d_0/configs/hyper_parameters.yaml' --num_iterations=8 --num_iterations_per_validation=4 --num_images_per_batch=2 --num_epochs=2 --num_warmup_iterations=4\n", - "2022-09-22 03:36:28,175 - INFO - CompletedProcess(args=['python', '/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/segresnet2d_0/scripts/train.py', 'run', \"--config_file='/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/segresnet2d_0/configs/transforms_validate.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/segresnet2d_0/configs/transforms_train.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/segresnet2d_0/configs/transforms_infer.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/segresnet2d_0/configs/network.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/segresnet2d_0/configs/hyper_parameters.yaml'\", '--num_iterations=8', '--num_iterations_per_validation=4', '--num_images_per_batch=2', '--num_epochs=2', '--num_warmup_iterations=4'], returncode=0, stdout=b\"[info] number of GPUs: 1\n", + "2022-10-18 08:28:21,861 - INFO - Launching: python /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/segresnet2d_0/scripts/train.py run --config_file='/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/segresnet2d_0/configs/transforms_validate.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/segresnet2d_0/configs/transforms_train.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/segresnet2d_0/configs/transforms_infer.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/segresnet2d_0/configs/network.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/segresnet2d_0/configs/hyper_parameters.yaml' --num_iterations=8 --num_iterations_per_validation=4 --num_images_per_batch=2 --num_epochs=2 --num_warmup_iterations=4\n", + "2022-10-18 08:28:28,958 - INFO - CompletedProcess(args=['python', '/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/segresnet2d_0/scripts/train.py', 'run', \"--config_file='/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/segresnet2d_0/configs/transforms_validate.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/segresnet2d_0/configs/transforms_train.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/segresnet2d_0/configs/transforms_infer.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/segresnet2d_0/configs/network.yaml','/workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/segresnet2d_0/configs/hyper_parameters.yaml'\", '--num_iterations=8', '--num_iterations_per_validation=4', '--num_images_per_batch=2', '--num_epochs=2', '--num_warmup_iterations=4'], returncode=0, stdout=b\"[info] number of GPUs: 1\n", "[info] world_size: 1\n", "train_files: 8\n", "val_files: 4\n", @@ -493,55 +495,55 @@ "----------\n", "epoch 1/2\n", "learning rate is set to 0.2\n", - "[2022-09-22 03:36:24] 1/4, train_loss: 0.6866\n", - "[2022-09-22 03:36:24] 2/4, train_loss: 0.6381\n", - "[2022-09-22 03:36:24] 3/4, train_loss: 0.5376\n", - "[2022-09-22 03:36:24] 4/4, train_loss: 0.5199\n", - "epoch 1 average loss: 0.5955, best mean dice: -1.0000 at epoch -1\n", - "1 / 4 tensor([[0.5086]], device='cuda:0')\n", - "2 / 4 tensor([[0.5086]], device='cuda:0')\n", - "3 / 4 tensor([[0.5086]], device='cuda:0')\n", - "4 / 4 tensor([[0.5086]], device='cuda:0')\n", - "evaluation metric - class 1: 0.5086382031440735\n", - "avg_metric 0.5086382031440735\n", + "[2022-10-18 08:28:25] 1/4, train_loss: 0.6673\n", + "[2022-10-18 08:28:25] 2/4, train_loss: 0.5865\n", + "[2022-10-18 08:28:26] 3/4, train_loss: 0.3893\n", + "[2022-10-18 08:28:26] 4/4, train_loss: 0.3853\n", + "epoch 1 average loss: 0.5071, best mean dice: -1.0000 at epoch -1\n", + "1 / 4 tensor([[0.5780]], device='cuda:0')\n", + "2 / 4 tensor([[0.5780]], device='cuda:0')\n", + "3 / 4 tensor([[0.5780]], device='cuda:0')\n", + "4 / 4 tensor([[0.5780]], device='cuda:0')\n", + "evaluation metric - class 1: 0.5780409574508667\n", + "avg_metric 0.5780409574508667\n", "saved new best metric model\n", - "current epoch: 1 current mean dice: 0.5086 best mean dice: 0.5086 at epoch 1\n", + "current epoch: 1 current mean dice: 0.5780 best mean dice: 0.5780 at epoch 1\n", "----------\n", "epoch 2/2\n", "learning rate is set to 0.0125\n", - "[2022-09-22 03:36:26] 1/4, train_loss: 0.4617\n", - "[2022-09-22 03:36:26] 2/4, train_loss: 0.4279\n", - "[2022-09-22 03:36:26] 3/4, train_loss: 0.4322\n", - "[2022-09-22 03:36:26] 4/4, train_loss: 0.4184\n", - "epoch 2 average loss: 0.4350, best mean dice: 0.5086 at epoch 1\n", - "1 / 4 tensor([[0.5543]], device='cuda:0')\n", - "2 / 4 tensor([[0.5543]], device='cuda:0')\n", - "3 / 4 tensor([[0.5543]], device='cuda:0')\n", - "4 / 4 tensor([[0.5543]], device='cuda:0')\n", - "evaluation metric - class 1: 0.5543338060379028\n", - "avg_metric 0.5543338060379028\n", + "[2022-10-18 08:28:27] 1/4, train_loss: 0.4148\n", + "[2022-10-18 08:28:27] 2/4, train_loss: 0.3646\n", + "[2022-10-18 08:28:27] 3/4, train_loss: 0.6224\n", + "[2022-10-18 08:28:27] 4/4, train_loss: 0.4244\n", + "epoch 2 average loss: 0.4565, best mean dice: 0.5780 at epoch 1\n", + "1 / 4 tensor([[0.5818]], device='cuda:0')\n", + "2 / 4 tensor([[0.5818]], device='cuda:0')\n", + "3 / 4 tensor([[0.5818]], device='cuda:0')\n", + "4 / 4 tensor([[0.5818]], device='cuda:0')\n", + "evaluation metric - class 1: 0.581782341003418\n", + "avg_metric 0.581782341003418\n", "saved new best metric model\n", - "current epoch: 2 current mean dice: 0.5543 best mean dice: 0.5543 at epoch 2\n", - "train completed, best_metric: 0.5543 at epoch: 2\n", - "0.5543338060379028\n", + "current epoch: 2 current mean dice: 0.5818 best mean dice: 0.5818 at epoch 2\n", + "train completed, best_metric: 0.5818 at epoch: 2\n", + "0.581782341003418\n", "\", stderr=b'')\n", "0\n", - "[info] checkpoint /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/dints_0/model_fold0/best_metric_model.pt loaded\n", - "2022-09-22 03:36:29,938 INFO image_writer.py:194 - writing: /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/dints_0/prediction_testing/test_image_001/test_image_001_seg.nii.gz\n", - "[info] checkpoint /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/segresnet2d_0/model_fold0/best_metric_model.pt loaded\n", - "2022-09-22 03:36:30,291 INFO image_writer.py:194 - writing: /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/segresnet2d_0/prediction_testing/test_image_001/test_image_001_seg.nii.gz\n", + "[info] checkpoint /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/segresnet_0/model_fold0/best_metric_model.pt loaded\n", + "2022-10-18 08:28:30,541 INFO image_writer.py:193 - writing: /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/segresnet_0/prediction_testing/test_image_001/test_image_001_seg.nii.gz\n", + "[info] checkpoint /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/segresnet2d_0/model_fold0/best_metric_model.pt loaded\n", + "2022-10-18 08:28:30,888 INFO image_writer.py:193 - writing: /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/segresnet2d_0/prediction_testing/test_image_001/test_image_001_seg.nii.gz\n", "1\n", - "[info] checkpoint /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/dints_0/model_fold0/best_metric_model.pt loaded\n", - "2022-09-22 03:36:31,530 INFO image_writer.py:194 - writing: /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/dints_0/prediction_testing/test_image_002/test_image_002_seg.nii.gz\n", - "[info] checkpoint /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/segresnet2d_0/model_fold0/best_metric_model.pt loaded\n", - "2022-09-22 03:36:31,873 INFO image_writer.py:194 - writing: /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/segresnet2d_0/prediction_testing/test_image_002/test_image_002_seg.nii.gz\n", + "[info] checkpoint /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/segresnet_0/model_fold0/best_metric_model.pt loaded\n", + "2022-10-18 08:28:31,645 INFO image_writer.py:193 - writing: /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/segresnet_0/prediction_testing/test_image_002/test_image_002_seg.nii.gz\n", + "[info] checkpoint /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/segresnet2d_0/model_fold0/best_metric_model.pt loaded\n", + "2022-10-18 08:28:32,019 INFO image_writer.py:193 - writing: /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/segresnet2d_0/prediction_testing/test_image_002/test_image_002_seg.nii.gz\n", "Auto3Dseg picked the following networks to ensemble:\n", - "dints_0\n", + "segresnet_0\n", "segresnet2d_0\n", - "2022-09-22 03:36:31,881 INFO image_writer.py:194 - writing: /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/ensemble_output/test_image_001/test_image_001_ensemble.nii.gz\n", - "2022-09-22 03:36:31,888 INFO image_writer.py:194 - writing: /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/ensemble_output/test_image_002/test_image_002_ensemble.nii.gz\n", - "2022-09-22 03:36:31,894 - INFO - Auto3Dseg ensemble prediction outputs are saved in /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/auto3dseg_work_dir/ensemble_output.\n", - "2022-09-22 03:36:31,895 - INFO - Auto3Dseg pipeline is complete successfully.\n" + "2022-10-18 08:28:32,025 INFO image_writer.py:193 - writing: /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/ensemble_output/test_image_001/test_image_001_ensemble.nii.gz\n", + "2022-10-18 08:28:32,032 INFO image_writer.py:193 - writing: /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/ensemble_output/test_image_002/test_image_002_ensemble.nii.gz\n", + "2022-10-18 08:28:32,038 - INFO - Auto3Dseg ensemble prediction outputs are saved in /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/helloworld_work_dir/ensemble_output.\n", + "2022-10-18 08:28:32,039 - INFO - Auto3Dseg pipeline is complete successfully.\n" ] } ], @@ -563,7 +565,7 @@ "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEVCAYAAADn6Y5lAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAA6IklEQVR4nO2de5wU5ZX3v6d7GBhGFGFG5CrDRUF9jUkI4OKFVWPwBmqyLuzqesuL7sbc0EQjm7wqu3nJrgbNJ3k3IeIt0agxiiQYjEYRTVBGs+4ooHHEMSC3GcQIA8Mw3ef9o6p7asaeme6Z6qrq7vP9fOoz1VVPV53ufn5nnufU8zxHVBXDMAyjdIiFbYBhGIYRLOb4DcMwSgxz/IZhGCWGOX7DMIwSwxy/YRhGiWGO3zAMo8Qwx58lIrJeRGaGbYdhFBIi0iAiZ2ZRTkVkQi/v0ev3liplYRtQKKjqcWHbYBiG4QfW4jcMwygxzPFnSarLKiI3i8gvReTnIrJHRF4XkaNF5FsislNENovIWZ73XSEiG92ym0Tk6k7X/aaIbBORrSLyRW+3VUT6i8htIvIXEdkhIj8WkYqgP7th9BURmSoia0XkQ7e+/1BEyjsVO8fVSJOI/KeIxDzvv9LV0W4ReUpEjgr4IxQV5vh7x/nAz4DDgf8GnsL5LkcCtwI/8ZTdCZwHHApcASwRkU8BiMgsYAFwJjABmNnpPouBo4ET3fMjge/k4fMYRr5JAF8HqoCTgDOAf+lU5kJgCvApYA5wJYCIzAFuAi4CqoEXgF8EYnWRIrZWT3aISAPwReBkYIaqftY9fj5OJTxMVRMiMgj4CDhcVT/McJ3lwHOqeqeI3A3sUNVvuecmAG8DE4F3gL3ACar6jnv+JOBBVa3J52c1DL9I6UZVn+l0/GvAaap6oftagbNVdZX7+l+Az6vqGSLyW+BRVV3mnovhaGOyqr7nvneiqtYH9bkKHWvx944dnv39QJOqJjyvAQ4BEJGzReQlEflARD4EzsFp9QCMADZ7ruXdrwYGAq+63eMPgVXuccMoKNxw6G9EZLuIfAR8l3YdpPDW//dw9AFwFHCnRwcfAILTAzZ6gTn+PCIi/YFfAbcBw1R1MPAkTqUF2AaM8rxltGe/CeefyHGqOtjdDlPVQ/JvuWH4zn8Bb+K0zA/FCd1IpzLe+j8G2Orubwau9uhgsKpWqOof8251kWKOP7+UA/2BRqBNRM4GzvKcfwS4QkQmi8hA4NupE6qaBH6K80zgCAARGSkinwvMesPwj1QIdK+ITAL+OUOZb4jI4SIyGvgq8LB7/MfAt0TkOAAROUxE/i4Io4sVc/x5RFX3AF/BcfC7gX8AVnjO/xb4AfAcUA+85J464P69IXXc7R4/AxwTiPGG4S/X49T/PTgNmoczlHkCeBV4DVgJLANQ1ceB7wEPuTp4Azg7/yYXL/ZwN0KIyGScSt1fVdvCtscwjOLEWvwhIyIXuuP1D8dp1fzanL5hGPnEHH/4XI0z1v8dnLHOmWKfhmEYvhG64xeRWSLylojUi8iNYdsTNKo6yx2tM0RVL1TVbWHbZESPUteJ4S+hxvhFJA78GfgssAWoBeap6obQjDKMiGE6Mfwm7NU5pwL1qroJQEQewpmq3WWFLpf+OoDKgMwzOrOH3U2qapPIgiUnnZhGwqUQNBK24x9Jx9l6W4BpnQuJyHxgPsAABjJNzgjGOuNjPKOPvhe2DSVIjzoxjUSHQtBI6DH+bFDVpao6RVWn9KN/2OYYRuQwjRi5ELbjf5+O07RHuccMw2jHdGL4StiOvxaYKCI17trcc/HMbDUMAzCdGD4TaoxfVdtE5Fqc9ezjwN2quj5MmwwjaphODL8J++EuqvokzoqVhmF0genE8JOwQz2GYRhGN4jI3W5a1ze6OC8i8gN3cl9dKsNfd5jjNwzDiDb3ArO6OX82Tta+iThDev+rpwua4zcMw4gwqroGJ+tYV8wB7leHl4DBIjK8u2uGHuMvRfZdNI2tp7YnH4q3COMX1ZFsbg7Rqr7zub8dqE0fJNOv/1R34ClV7a6lYhgZKVaNQEed/KnuwHqgxXN6qaouzfGSmSb4jcTJ8JcRc/wBERs0iNghzjT6LWcp787+SfrcxtZ9LPj5lZTt+hBNJknsbIQCzJOw84M2nv/tkenXh43c3DmnqmF0SSloBDrq5LCRm1tUdUrQNpjjD4j6hcezYu7tAFTHFTxrqRzdbwB3rlxGQoUNrUeybMY0Eo2NIVnaexQ4QKLHcoaRiVLQCORFJzlP8LMYf56JVVayafFJnDrzdSaXD2Ry+UCq4h0X0IpLjKP7VTK5fCCnVWxj46Ia9l8wNSSLe08SpUWT6c0wsqGUNAIddeITK4B/ckf3TAf+2tPy7tbizyOxQYOgZiQr593G0f2yWy2xKl7Ju7OXUpOcz+S1RxRUlzapwr6k9FzQMFxKTSOQu05E5BfATKBKRLYA/wfoB6CqP8aZ33EOTn7ufcAVPV3THH8eqV94PCvn3cb4soqc31t7/hKeP3N4QXVpFWjReNhmGAVEqWkEcteJqs7r4bwCX8rFBnP8eSQ5QLNuxXSmKl7JpPIdECucFnQSoUWtShnZU2oagWjoxFSaD0QoG3YEyQF9i+HFULR6CLG9zQUxjC2J0KzlYZthFAIlqhGIhk7s4W4eiB9RzZVr1lJ77pI+XSc1kuGdhSf4ZFl+SarQov3Sm2F0RalqBDrqJCysxZ8HRIRjy7d/bGRCrqRGMiT7F8aDK0VoSZrDN3qmVDUC0dCJOX7DNxLEaE5a9ifD6I4o6MQcv+EbqsIBC/EYRrdEQScW488DyT17mfPgdVzxl1P6dJ2mRDPjHr+aEWsKYzJU0u3CpjbD6IpS1Qh01ElYmOPPA8nmZmpuWsual47r03V2JGJMuvVdKp5Y55Nl+SWJsC/ZP70ZRleUqkago07CwkI9hm+ohv/QyjCiThR0Yi3+PDJydZKa5fNpSuQ+vviShplc9MACdG9hjE0GpyVzIFmW3gyjJ0pNI9BRJ2Fhjj+PVCxfx+RbGnh2/4isK3ZCk6xv3U/tc5MZu3BtwUxKgdQaJOXpzTB6otQ0Ah11Ehbm+PNMYmcj986YymdWfj2r8n8+2MI3zr2ccd+ty7Nl/qPW4jd6QSlpBDrqJCwCubOIjAbuB4bhrFG0VFXvFJEhwMPAWKABuFhVdwdhU2CokmhsZPSqGiY0XwPAKTPWc8+YF9JFmhLNTF2xgFiLEG8RxjUUZqYhRWg1h99rSlYnJaQRiIZOgrp7G3Cdqv5JRAYBr4rI08DlwO9VdbGI3AjcCNwQkE2BUrF8HeOXO/t/WHwSdXOfTp/bcGAUk27elF5hsHAGpnUkqcL+hD3c7QMlrZNS0AhEQyeBOH43KcA2d3+PiGzEyQk5B2edaYD7gNUUYYXuzPhFddy4ZHb7gaSSaGoKzyCfUBVaE9bi7y2mk3aKVSMQDZ0EfncRGQt8EngZGObJFLMdp4ub6T3zgfkAAxgYgJVdExs4kPe+fmKXa4MM/2Mb5atqu71GsrkZCrSb2h1JoDVp6/H7Qa46iZRGKiup//YJJAdk1sjI1Ukqlnc/7r5YNQLR0Emgjl9EDgF+BXxNVT8SaV9HW1VVRDLWFDfr/FKAQ2VIaKsxxSorkTEjmP8PTzKx//aMZb7e7womvFrttE4KKCuQH6gKLdbi7zO90UlkNJJFRq2a8sLMnOUXUdBJYKN6RKQfTmV+QFUfcw/vEJHh7vnhwM6g7OkN733tE5z1aC3jyrs286YLf8W4Jz8iPuTwAC2LBkmE1kQ8vRm5U+g6qV94PHf+elm3GbVqz1/CVS+8RLyqKkDLooNXJ2ERiOMXp8myDNioqt/3nFoBXObuXwY8EYQ9uRIbOJDNC/+Gw07ewcT+24lL14+Wqss+Ysqgd3n7G8fQOuszAVoZPqpwMBlPb0ZuFLJOvAnTj+5XSVy6di1V8cqCT5jeF7w6CYugWvwzgEuB00XkNXc7B1gMfFZE3gbOdF9HDqms5IvzVvGvE1dmVb667CPu+MI9bJ9WWiNcFKG1rSy9GTlTsDqJDTqEFXNvZ9mYF7Mqn0qYvvWU0ptK5NVJWAQ1qudFoKvEmGcEYYORf1ShLVl6QvYL00lpEAWdmEoNHxHaErH01mNpkVki8paI1Lvj0zufv1xEGj2t3y/mxWzDCBTJWiP5wvrjhm+owsG27OKWIhIHfgR8FtgC1IrIClXd0Know6p6rb+WGkZ45KKTfGEtfsM3VIVEMpbeemAqUK+qm1S1FXgIZ6KSYRQ1Xp2EhTl+w1eSCUlvQJWIvOLZ5nuKjgQ2e15vcY915vMiUicij7pr2RhGwePRSChYqCcbNEn9/iMYUraX6rKPeiye0Bj1B44kfiAA2yKEKiTbOrQlmlR1Sh8u+WvgF6p6QESuxlmu4PS+2GjkB1Xl9dbhVMe3UhXPPHHLS0KTvHnwAPGW8JxfWGTQSeBYiz8LEk27aJg1kH9f/vmsytcfOJJnLvoko3/wWn4NixoqJBOx9NYD7wPeFvwo91j75VR3qWrq3+ddwKd9s9XwlVJbWrlPeHQSFub4sySx6wNGrW7j67+8gsa2Q7ssd/Obs1n2wCx0yzaS+/YFaGFESEr71j21wEQRqRGRcmAuzkSlNKnZqi6zgY2+2mr4R3ppZXrMqHVJw0wueOA6tGFLwS6t3Gey00jesFBPDpSvqmXCK0NZd+Y4aioaM5bZ/2IVoxb/saCXje01CtqWXWVW1TYRuRZ4CogDd6vqehG5FXhFVVcAXxGR2TjLFX+AszyxEWEqlq9j8tojePbMEUwq35GxTDpzVsC2RQYFDTG+D+b4c8YJ+wyhQcZmPD9632slXaElhwqtqk8CT3Y69h3P/reAb/lmnxEIqbAPscx1YdzeutLVCDg6ybKBBM58F+BOnAbSXaq6uNP5y4H/pD1U+kNVvau7a5rj7wWJXR+EbUJEEQi5JWNEADfsY3RF9jrJ13wXi/Eb/uG2ZFKbYRgZcHvGWfaO8zLfxRy/4SuSlPRmGEZmPI6/u7kukKf5LhbqMfxDQRJhG2EYEaejTvo61wV6Md/FWvyGb4hboVObYRgfx6uTLMjLfBdz/IavWIzfMHomB43kZb6LhXoM/7BQj2H0TA46ydd8F3P8hq90k5XSMAyXXHSSj/ku5vgN3xCFWFvYVhhGtImCTszx98D+C6Z2mRc03iKM+25d6a430hkL9ZQk+y6axtZTM8er4y3C+EWmkQ5EQCfm+DMQq6wkNugQADafBa/NXpKx3MbWcm598B8o2/UhqkpiZ6Oz5moJE3aFNoIhNmgQsUOc5Ze3nKW8O/snGcttbN3Hgp9f6WgkmTSNuIStE3P8GXhn4QncP/eHAFTH9+M8U/k4x/Q7wM2/+TlJjbH+wEh+der/Ku2p6hHowhrBUL/weFbMvR2A6rgCmdfgP7rfAO5cuYyEChtaj2TZjGmlrRGIhE4CdfzuuhOvAO+r6nkiUoMzBXko8CpwqTstOVSS/ZXJ5Skzus6NGRPhmH7OL5hgK8ROCMC66CJqD3f7SsFoZIAyuXxgj+XiEuPofs4/hSQ7uly4rZSIgk6CHsf/VTqOMf0esERVJwC7gasCtqcjIsSHHUGiIvdfJY5C1eHEKnvOPlTM2ASuPhN5jZQdOYzkgNw1EkPR6iElrxEIXyOBOX4RGQWcizOzDBERnGnFj7pF7gMuCMqeTMSrqrj4+dd45tzv5/zeVNhn040l3Op3u7CpzciNgtDIEdVcuWYttedmfu7VHamwzzsLS1gj0EEnYRFki/8O4JuQXop7KPChqqY+fleLDyEi81OLGB0kj4lsY8Kk8m1UxbsO73T9Vifsk+yfB7sKBVuyoa/cQcQ1IiIcW749q7y6nUmFfZL9S/zhbm5LNuSFQGL8InIesFNVXxWRmbm+X1WXAksBDpUhka41iQFJ4sOOyHwyqSSamop2VINgDr+3lJRGKpKUDT8y47lSGPkTBZ0E9XB3BjBbRM4BBgCH4mSUGSwiZW6L5mOLDxUij53/A96ZVZ3x3Fstw3nx9FEkmnYFbFVAKMTaileweaZkNFJ7/hIaz8n8kLckRv5EQCeBOH7vlGK3NXO9qv6jiPwS+ALOqIXLgCeCsCefVMUP0k8y5xodGt/LsoV/y+hnxtF/ZW3AlgWAQsxa/L2itDRSSVUX0dTq+DauX1TD6FU1VCxfF6xhQREBnYS9OucNwAIRqceJZy4L2Z68Uh1vZsWFS9hyepx4dTVIcQ1tc7qwmt4MXygpjVTFK3l39lI2n4UTMi0yjUBHnYRF4BO4VHU1sNrd34STWqykuPvCH/PiZ48pvrCPQuxg2EYUPqYRJxz0/JnDizPsEwGdhN3ijxS6t5nLHriW/91wXq/evyeZ4EAW/8SHxvbzN5Vv89bCiRw49zO9ulckUYglNL0ZxUdyz17mPHgdV/zllLzepypeyWkV29i4qIb9FxTZ/z2PTsLCHL+HZHMzY7+9lj+9PDH396qyJxmjRbMbCpoK+2ydUTyrZoiqhXqKnGRzMzU3rWXNS8fl/V6psE9XiyQWKl6dhEXxeB0jfBRiB83hG0a3REAn5vgzMGJNkimJBQB88m/+zE/H/qbLsnuSzuN5Z8ZNcbVMeoO19EuDkauTTEhcA8ApM9Zzz5gXQraosAhbJ+b4M1DxxDrGuYPm/uffT+J/RlSkz5WTYEK/FuI4ow32JGMkKb6RB71B3NilUfxULF/H+OXO/h8Wn0Td3KfT5/qR5Oh+A4iLNYQyEQWdmOPvgXHfrWPxD85qPzB0MNc88Rsm9msKz6iooooctOU5S43xi+q4ccns9gNVh3PHk3enV+U0OhEBnZjj74FkczN4sgfFgYNaZq38TERgRqIRPJ01UiZCQk0fXRIBnZjjN3xFEtbiN4yeCFsn5vgN3xBV5KCt2WAY3REFnZjjN/xDQdqsxW8Y3RIBndhj91xJKhtbRtCY6NuDqwTCm63DKNtfRLFQVUgk2jejJFFVXm8dTlOiuefC3ZDQJOtb9xNvKSKNQEedhIQ5/hxJNDXxx9NHctWK+X26ztutR7Bszuc46vbX/DEsCihIayK9GaVJYmcj986YymdWfr1P1/nzwRa+ce7ljPtunU+WRQSPTsLCHH+uqJJo2sXoZxLM+dXX2ZWs6Pk9nfhK/d/z7Z9dgr73Psl9+/JgZFhYi9/A0UhjI6NXQc3y+b1q+V/SMJMLHrgObdjijBoqKqzFX7D0X1nL0YvfYc3eSVmHfVLhnW3Pj2L0oj8WmdPH6cK2tbVvRklTsXwdk29p4Nn9I7J2/qnwTu1zkxm7cG0ROn066iQkzPH3gVzDPkUZ3vGiCgfb2jej5Mk17FO04R0vXp2EhI3q6QvpsM84Lty/oMfiZfuFo957rfha+ikUa+kbHUmHfWqY0HxNj8XjLcK4hrribOmniIBOzPH7QP+VtdSszK5sUQ92VEWtpW9kwLu2T08UtUYgEjqxUI/hH6poa2t66wkRmSUib4lIvYjcmOF8fxF52D3/soiMzYfZhhEoHp1kQz50Yo7f8I8cHu6KSBz4EXA2cCwwT0SO7VTsKmC3qk4AlgDfy4PVhhEsOTzczZdOzPEbvqGqJA+2pbcemArUq+omVW0FHgLmdCozB7jP3X8UOEOkCLNvGyWFVydZkBedWIzf8I097H7qmeQjVZ5DA0TkFc/rpaq61N0fCWz2nNsCTOt0yXQZVW0Tkb8CQwFbE9soWDrppDuNQJ50EpjjF5HBwF3A8TjPta8E3gIeBsYCDcDFqro7KJsMf1HVWWHbUOiYToqfKOgkyFDPncAqVZ0EfALYCNwI/F5VJwK/d18bpcH7wGjP61HusYxlRKQMOAzYFYh14WE6MbzkRSeBOH4ROQw4FVgGoKqtqvohHWNT9wEXBGGPEQlqgYkiUiMi5cBcYEWnMiuAy9z9LwDPqmrRZnoxnRgZyItOgmrx1wCNwD0i8t8icpeIVALDVHWbW2Y7MCzTm0Vkvoi8IiKvHORAQCYb+URV24BrgadwWrWPqOp6EblVRFJ5/JYBQ0WkHlhA8bd0e60T00hxki+dSBANKBGZArwEzFDVl0XkTuAj4MuqOthTbreqHt7dtQ6VITpNzsirvUbXPKOPvqqqU8K2oxjxSyemkXApBI0E1eLfAmxR1Zfd148CnwJ2iMhwAPfvzoDsMYwoYjoxAiEQx6+q24HNInKMe+gMYAMdY1OXAU8EYY9hRBHTiREUQY7j/zLwgPuAYhNwBc4/nkdE5CrgPeDiAO0xjChiOjHyTiAxfj8RkUagmehN4qkiejaB/3YdparVPl7P8JkIawSiqZOS00jBOX4AEXklag9PomgTRNcuI79E9XePol1RtCnf2Fo9hmEYJYY5fsMwjBKjUB3/0p6LBE4UbYLo2mXkl6j+7lG0K4o25ZWCjPEbhmEYvadQW/yGYRhGLzHHbxiGUWIUlOPvKfdkQDaMFpHnRGSDiKwXka+6x28WkfdF5DV3OycE2xpE5HX3/q+4x4aIyNMi8rb7t9u1kIzCx3TSo20lr5OCifG7uSf/DHwWZ02TWmCeqm4I2I7hwHBV/ZOIDAJexVkm92Jgr6reFqQ9nWxrAKaoapPn2H8AH6jqYtcJHK6qN4Rlo5FfTCdZ2dZAieukkFr82eSezDuquk1V/+Tu78FZKnVk0HbkgK3lXlqYTnpHSemkkBx/ptyToVYkERkLfBJIraZ4rYjUicjdIXUVFfidiLwqIvPdY1nlPDCKBtNJz5S8TgrJ8UcKETkE+BXwNVX9CPgvYDxwIrANuD0Es05W1U8BZwNfEpFTvSfdrDyFEdszigLTSTQpJMefTe7JQBCRfjiV+QFVfQxAVXeoakJVk8BPcbrcgaKq77t/dwKPuzbYWu6lhemkB0wnheX4s8k9mXdERHBSnW1U1e97jg/3FLsQeCNguyrdh2i46frOcm2wtdxLC9NJ93aZTgh2Pf4+oaptIpLKPRkH7lbV9SGYMgO4FHhdRF5zj90EzBORE3G6iA3A1QHbNQx43NEbZcCDqrpKRGqxtdxLBtNJj5hOKKDhnIZhGIY/FFKoxzAMw/ABc/yGYRglhjl+wzCMEsMcv2EYRolhjt8wDKPEMMdvGIZRYpjjNwzDKDF6dPzu2tVn+nlTEZkpIlv8vGYQiMhYEVERyWnim4hcLiIv5suuoPHWCRG5SUTu6uV11ovITD9tCxsROcZd532PiHwlbHuCwrThUCjaKJiZu0Y0UdXvZlNORO4Ftqjqv3ree1y+7AqRbwLPqeqJYRtihEuUtVHwoR438YTRS3JtoRk9chTQqyUS/P4tTBt9o6i1oardbjjraXwL2ADsBu4BBrjnDgd+AzS6534DjPK8d4hbfqt7frl7fCbOf7hUua+41x/lvv4mzpKtW4Ev4qzrMcE9dy/O0q5PAs3AmcBkYDXwIY7oZnuuvRr4ouf15cCLntcKXAO87b7/R7QvZREHbgOagE3Al9zyZV18V6OBx9zvYxfwwy7ueSfOmukf4WQmOsVzbirwintuB/B99/gA4OfudT/EWYxrWC9+s5k4a7TfgLPu+M9wGgA3Au+4138EGOK53qU465fsAha61z/TPXcz8HNP2ZOBP7o2bnY/+3zgINAK7AV+7bEzdZ3+wB3ub77V3e/fyebrcFZN3AZc0VPdDXoDngUSQIv7OY8GDgPud+vEe8C/AjFPvfgDsMT9bv/N/R5uA/7i/v4/Bio89zBtmDbuoI/ayNbxv+H+cENwKuq/ueeGAp8HBgKDgF/iOnf3/ErgYZx/EP2A07zGuvvfAf4EVLuvZ7lf+nHudX/Oxyv3X3EWgYq5963HWQCqHDgd2AMck0Pl/g0wGBiDUzFnueeuAd70fPbn6KJy4wjhf3BEXIlTGU/u4p6XuN9dmfuDbae98q0FLnX3DwGmu/tXA792v5M48Gng0F78ZjOBNuB7OBWqAvgq8BLOEr79gZ8Av3DLH4tTIU91z33fff/HKjdOa3cPMM/9vYcCJ3p+t3/LYGfqOre6NhwBVOMIZFEnm291r3sOsA8nPV7oDr/TZ+pc3+7HWelxEDAWJy3iVZ560QZ82a0LFW79WeH+boPc3/z/mjZMG/iojWwd/zWe1+cA73RR9kRgt7s/HEhmMsA19n33i3oROMxz7m7ciu6+nsDHK/f9nvOnuJUj5jn2C+DmHCr3yZ7XjwA3uvvPdvrsZ9F15T4JRxiZznW4Z4bzu4FPuPtrgFuAqk5lrnR/8BP68pu5330rrpjcYxuBMzyvh+O0Qspw/jE/5DlX6b4/U+X+FvB4FzbdS/eV+x3gHM+5zwENHpv3e79bnNbN9J6+i6A3b33DcUKtwLGe81cDqz314i+ec4LTUh/fqV69a9owbfipjWxj/N5Ubu8BIwBEZKCI/ERE3hORj9wfZrAbWxyNk7x4dxfXHIzTzfm/qvpXz/ERne7n3c90bASwWZ3EDl4bc0k3t92zvw+nNZHJlve6ucZo4D1VbevpZiJyvYhsFJG/isiHOOGAKvf0VTghgjdFpFZEznOP/wxnqd2HRGSriPyHm+iiKzL+Zi6NqtrieX0UzlK1H7r2bMQJWQyj03egqs043dpMjMappL1hBB2/38427+r03Xp/p6hShdMK6/y5vHXT+ztV47RaX/X8Fqvc42DaMG1ktjlnbWTr+L0ZfcbgxJnA6YodA0xT1UNxujzgtFw2A0NEZHAX19wNnAfcIyIzPMe34XSrMt07hXr2twKjRcT7WcbQnnWoGUdMKY7swp5MbOPjn70rNgNjenogJCKn4MRpL8bpDQ3G6Z4LgKq+rarzcLp13wMeFZFKVT2oqreo6rHA3+B8d//Uza26+s3g42nlNgNnq+pgzzZAnUxFHb4DERmI003NxGactHqZ6HzPzmzFEVlXNhciTTitw86fy5sRSzuV3w8c5/kdDlPVlIhNG6aNTDbnTLaO/0siMkpEhuA8wHjYPT4Ip6J+6J77P6k3qJO4+LfA/xORw0WkX4bclquBfwQeE5FUCrZHgCtEZLL7RX67B9texvkP9033HjOB84GH3POvARe5vZMJOK2GbHkE+Ir72Q/HecjTFetwKsJiN8vPgE7/0FIMwonJNQJlIvId4NDUSRG5RESq3Vbah+7hpIj8rYj8L7c39RGOQ0nSNV39Zpn4MfDvInKUa0O1iMxxzz0KnCciJ4uT0elWuq43DwBnisjFIlImIkPdpBvgPIwb140NvwD+1b13FU43+ufdlI88qprAqUP/LiKD3O93AV18Lm1PR7hERI4AEJGRIvI5t4hpw7ThizaydfwPAr/DeXr/Ds7oA3CeLlfgtFRewumWerkU50d4Eyfu9LXOF1bVp3FidL8WkU+p6m+BH+A8LKp3rwtwIJNhqtqKU5nPdu34f8A/qeqbbpElOHG3HcB9OD9AtvwUpwv5PzgPoB/rqqAr8vNx4q5/wXnS/vcZij6F8z39GafL1kLHrucsYL2I7MUZ4TBXVffjtMYexanYG4Hncbq4XdHVb5aJO3EeKP5ORPbgfOfT3M+1HmfExoM44t3tfrZM38FfcGKm1wEf4DiWT7inlwHHul3m5Rne/m84IzbqgNdxvu/ubC4UvozTst6E8zzrQZxYfVfcgFvv3fDpMzi9akwbpg180kbkM3CJyGScp/D9s4kRGs7sQZyHds+EbYuRP0wbuWPacIjkBC4RuVBE+rtdyO/hjG21im2UPKYNww9Cd/wiMktE3hKRehFJxQmvxgkNvYPzBP2fQzPQMCJASic4sd2/Ytow+kCooR73Ycyfgc/ixMZqgXmquiE0owwjYphODL8Ju8U/FahX1U3ug6iHgDk9vMcwSg3TieErYS9CNJKOT+234D4x74qqIXEdO7q7uRlGPnm17kCTqlb3XNLwkZx0Ui79dQCVeTfKyMwedkdeI2E7/qwQkfk4s3wZM7KMdU9lmrdiBEF8eH13MzSNkPBqZAADmSZnhGxR6fKMPhp5jYQd6nmfjrPoRtFxViMAqrpUVaeo6pTqobbSrFFy9KgTr0b60T9Q44zCI2zHXwtMFJEad+bbXJzJEoZhtGM6MXwl1FCPqraJyLU4M/biwN3ubDjDMFxMJ4bfhB7jV9UncRJHGIbRBaYTw0/CDvUYJYyI3C0iO0XkjS7Oi4j8wJ3cVycinwraRsMIm3zoJPQWfylyScNMXlo76WPHT5mxnv8cuYqpKxZw2qc3cM+YF0KwLlDuBX6Ik6UqE2cDE91tGk5awW6H+xrFwb6LprH1VPnY8ZGrk1Q+vZ76b5/A8JcSVCxfF4J1gXMvPuvEHH9A7Ew0sz3hjEiqfW4y4xeu/ViZPyw+iWcufINJN29izcLjoMgdv6quEZGx3RSZg5NRSnFWqxwsIsPdJb+NIiM2aBCxQ5z5B1vOUt6d/ZOPlakpn8+kTaNZOe82zul3HeOXB2xkCORDJ+b4A2L6igVMuvVdAMbtrcu4WPj4RXXcf/sUEk1NdJ2zIbp87m8rddcHifTrV+sOrMdZWjfFUlVdmsMlM01cGomzBK5RZNQvPJ4Vc28HoDqukGESWu35S/jgXBhfVhGwdf7h1YkPGoFe6MQcv890FcYZtSZJYsfObt+bbG6G5mbA6dLWlM+n9vwlVMULYxZm4wdt/GFVe0a4gSMaWlR1SogmGRGkqzDOzOmvM7l8YIZ3tFMVr6TKncpzyoz1/GHxSYxfVOdop0Dw6iQsjZjj94mEJnnz4IEuwzi5Uvn7jUyqH8nzZw7ntIptBeH8FeWAvysEZzXBzygQRCgbdkSXYZxc+d7IVay+qI57fzaLWMOWgnH+UdCJjerxiT8fbOEb517OuO/W+XK9+oXHc/Xjv2HZBbP4zMqv+3LNfJME9mkivfnACuCf3FEL04G/Wny/cIkfUc2Va9ZSe+4SX643fcUC7r3gc8xfvpJ3Fp7gyzWDwKsTn8hZJ9bi94FLGmZS+9xkxjX41+Uc/lKCGxKXMu69OkavGsyE5msAp3sb1dE+SVVacljmW0R+AcwEqkRkC07O5n4AqvpjnHHr5+CkGdwHXOGzyUZA7LtoGlvOUk6reNy33utpn97A6q9M5uQBO5hx+husuWM64IRJozzaJwo6McffB7zhnbEL13ab3TlXKpavY+xyp3VQsXwd459wusmrKydHdrSPIrRo9p1IVZ3Xw3nFyWlqFCodwjtLyfTAtrfcM+YFVwuV3DPmBRKjn+fNgwe4qHkBY5f7dhvfiYJOzPH3ASe8c4XT0s/zvVLd5NMqHsNP8fhJAqFZbclso50g622QeuwLUdCJOf5eko/wTneICMeWb4/0Q15FaElalTIc8hHe6Y4kAk27I/+QNwo6MZX2kpfWTmK8z+GdQiepQou1+A2XraeKO3onuo2VMIiCTszxG76RRNintha8YXRHFHRijt/wDacLay1+w+iOKOjEHL/hG0nC78IaRtSJgk7M8Ru+kVShOWmhHsPojijoxBy/4RtJhAMW6jGMbomCTszxFwCxQYNIHnE4ccl+tl8YOBNTrEoZwbMz0czrB0ZBMtoagWjoxFRaANQvPJ6V826L/FK0SY2xL2GhHiN4pq9YwKSbN7lLmkebKOjEHH+EiVVWUv/tEzh15usc3S/6Y6GdLqxVKSM4mhLNTF2xgFG/UxKNjWGbkxVR0ImpNKLEBg2CmpGsnHdbQTh9AFVCr9BG6bAz0cyz+0Y5Lf0CcfoQDZ0EsiyziIwWkedEZIOIrBeRr7rHh4jI0yLytvv38CDsKQTqFx7Pnb9eFvnwjpckMfYnytObkRumk9yYvmIB9588pSDCO168OgmLoNbjbwOuU9VjgenAl0TkWOBG4PeqOhH4vfs60jQlmhn3+NWMWJOfxRpilZVsWnxSOrwTl8JJmZBqyaQ2I2eKQiepOjxz+ht5uX5Kg+nwTg5LHEcBr07CIpA7u0kBtrn7e0RkI05OyDk460wD3AesBm4IwqbesiMRY9Kt7/aYRjEn3KVrESFZPbigwjtekgityXjYZhQsxaKT2KBDWDH39h7TKOZCagn0BMKGA4UX3vESBZ0E/i/HzRb/SeBlYJgnU8x2YFjQ9kSB1NK1x5ZvJy5aUOEdL6pCS8LG8fuB6aQjqSWXadoNSS248I6XKOgk0DiCiBwC/Ar4mqp+5D3nJhPI2GcTkfki8oqIvNK4y7d0Zb1iWDzJxlvGsv+Cqb5cb99F09h4y1hOq9jG5PKBBRfe8ZJEaE3E05vRO3qjE69GDnIgIEszk9yzlzkPXscVfznFl+td0jCTCx64Dm3YQmLHzoIM73jx6iQsAvMwItIPpzI/oKqPuYd3iMhw9/xwIGP8RFWXquoUVZ1SPTRch1IVr+Td2UvZfBbE3fBMNsQGDUqXjw0aRNnwIykbfmQ6M1GU19nPFlVoTcbTm5E7vdWJVyP9CHeMeLK5mZqb1vKH1cezsXUfCc3uedjORHO6/M5EM3WtLdS1trRnuIv4OvvZ4tVJWAQ1qkeAZcBGVf2+59QK4DJ3/zLgiSDs8YPa85dw1QsvEa+qyqp8/cLj0+XrFx7PbWsf47a1j/mWeDoKKMKBRFl6M3Kj2HQyflEd1517Be+07c+q/PQVC9Llp69YwI3TZnPjtNmM+25dni0NFq9OwiKoO88ALgVeF5HX3GM3AYuBR0TkKuA94OKA7OkzX9t8rpOBa1/3ldI7Ceu0im1cv6iG00983dcHX1FBVThoIZ6+UFQ62fu549lyljKkh+aldxKWNmzhvAeuZ9TLCX8HUESIKOgkqFE9LwJdxUTOCMIGv8k2A1fHEQ6VbsLp4kQV2hKF+XwiChSbTrLNwOUdKZcEam5aG4h9YREFnVh/3PANdR9aGYbRNVHQiTl+wzeclow5fsPojijoxBx/Pug0ISvqyyn7hSIkLNRjZEHHCVkjCmI5Zb+Igk7M8eeBYpmQlTNK6BXaKAyKaUJWzkRAJ+b4e8kpM9az5o7pAIxcnaRi+br0ORHh2PLtRTlypztUIWmO33AZuTrJhMQ1gKOXe8a8kD6XRKBpd9GO3OmOKOjEHH8vuWfMC+BW5AmJaxj/RGmGdzoiJBPZTWgzip+K5esYv9zZX3PHdBKjny/Z8E5HwteJOX6fKNnwjhcFtRa/0QUlHd7xEgGdmOP3gVNmrGf1LZM5reLxolh6odcoYC1+IwMjVye5oPk6xjXUFc3SC70mAjoxx+8D7WGfEnb6LmqO38hAxfJ1jF1OjxMeS4WwdWL9csM/VJC29q0nRGSWiLwlIvUi8rHkIiJyuYg0ishr7vbFvNhtGEGi2WsE8qMTa/Eb/pFDF1ZE4sCPgM8CW4BaEVmhqhs6FX1YVa/11U7DCJMI6MRa/IavSLJ964GpQL2qblLVVuAhnExThlH0ZKkRyJNOzPEb/qF0DvVUpZKDuNt8T+mRwGbP6y3usc58XkTqRORRERmdR+sNIxg8OqF7jUCedGKhHsNXpGOCtCZVndKHy/0a+IWqHhCRq3HyzZ7eh+sZRiTw6KSvGoFe6MRa/IZviIIkJL31wPuAt2Uyyj2WRlV3qWoqj+BdwKd9M9YwQsKrkyzIi07M8Rv+oRBra996oBaYKCI1IlIOzMXJNJUmlW7QZTaw0U9zDSMUNGuNQJ50YqEew1eyfGCFqraJyLXAU0AcuFtV14vIrcArqroC+IqIzAbagA+Ay/NitGEETNg6McffAzsTzexKCEf3G8Cu5P70flxiHZaW7Ucyfbxk0Y/F+Lsvrvok8GSnY9/x7H8L+JZf5hn5ITZoEDKwgsTORmKHHJLeR7XDEuWaTLYfL2UioJMS9lLZ0TkBtDd5tLP2yOXcOG12TkmlixXJLdRjFAn1C4/nqhdeIl5V1WEf2tewum3tYx2OlzKSW6gnL1iLvwu6SgCd2k/2V+Itkl57JLa3mfMeuJ4Zp7+RXn72koaZvLR2EvDxZWmLlVxaMkZhE6uspP7bJ3DqzNc5rWIb1y+q4fQT2/djLeNJDkim17CqjjvHR6+qSS9jvu+iaWw91XnI2Xl582ImbJ2Y48/AzkQzz+4bxaSbN5FobOyQALpzMuhUqC7Z3EzNTWv5w+KTqJv7NAC1z01m/EKn7B8Wn8TGuU8Vdzgoxy6sUbjEBg2CmpGsnHcbR/erBCp5d/ZS96x333kNUBV3jtck53Psy0cCsOUsdROyQ035fCavPaL4w0ER0Emgjt+dfvwK8L6qniciNTgz0YYCrwKXurPTQmX6igWO0+/FsrHjF9Vx45LZAIzbW5f+xzB+UR3X/ewK7njyblcoRYiCWIinTxSKRuoXHs/Kebf1avnx2vOX0HiO08qvjiupfwy15y/h+TOHs2zGNBKNjX6aGy0ioJOgm55fpeNQo+8BS1R1ArAbuCpgezrQlGhm3ONXM+p36lS8XrQ6ks3NJHbsJLFjZ4flZ5PNzciuD0lo8a5eKUAs2b4ZvSLSGolVVrJp8UmcOvN1ju5X2avea1W8ksnlA5lcPrDDMuZV8Uomle+AWPFqBDrqJCwCc/wiMgo4F2eCASIiOLPLHnWL3AdcEJQ9mdiRiDHp1nepeKI04oy+43ZhU5uRG4WgkdigQ1gx93aWjXkxTDMKGw1fI0G2+O8Avkl7WHwo8KGqpjo9Xa1BYRQQNqqnT9yBaaQkCFsjgTh+ETkP2Kmqr/by/fNTixg17srfv8lh8SQbbxnL/gum5u0exYxYi7/X+KmRgxzo+Q29JLlnL3MevI4r/nJK3u5R7EgJtfhnALNFpAHnQdXpwJ3AYBFJPWD+2BoUKVR1qapOUdUp1UPjeTPy8FgFv5l1Jzs+k597qCqvtw6nrrWFja37SGiRBcIVJKHpzcgJ3zTSj/55MzK5bx8Tf7CJ1a9Nzsv1YyhaPYSy4UcSdyd+FR0enYRFIKN6vDPLRGQmcL2q/qOI/BL4Ak5Fvwx4Igh7uiKVDHpcQ11eUsQldjZy74ypzsOrqsOLcoSPhXh6R6FoJDUh67SKx8hHqtGj+w3gzpXLSKiwofXIoh3hE7ZOwh5QfgOwQETqceKZy8I0JolA0+78JYNWZ7RQYsfO9ESwYuoyi0Is0b4ZvhApjYgIx5Zv7zAax0/iEuPofs6on9MqtrFxUU3RhV69OgmLwB2/qq5W1fPc/U2qOlVVJ6jq33mWFi16UhO+1rx0XNim+IeFenzBNOKQmvC19ZSw26c+UyqhHqNEUIgdNIdvGN0SAZ0U2b/SvhHGqJ6Rq5PULJ9PUyJP4aUAESzUU+yEMarnlBnreXfxScQqi+N5mFcnYWGO30MYXcuK5es49tb3aMwuG0+0UbVQT5ETRojynjEv8Pjc7yOHFIfj9+okLCzUY/hHBLqwhhF5IqATa/FnIMiu5b6LprHh5qPcxaoKHHu4WzIEGaK8pGEmFz2wAN1b+OFQIBIPd83xZ+CeMS/wxNzboWZ03ieSbD1VeHf20rwNjwsSAWJtyfRmFC8Vy9cx+ZYGnt0/Iu8TEl9aO4mxC9fmb5h1wHh1EhYW6umCUplI4iuqiDn8kqEUJiTmhQjoxFr8XVAKE0l8R0HaNL0ZRU6RT0jMGx6dhIU5/ixIjfbZfBa+h33iLVI86/YoSFsyvRmlQzoD3erjfa/PiYpkca3b49FJWJjjz4Ha85f4njB6/KK6oknULqrIwUR6M0qPfNTnfOguTLw6CQtz/DlQFa/0PeyTbG4uom6yIolEejNKj3zU53zoLlw0dI2Y48+RfEzyymc3OVAs1GOQn0le+Qy3Bo6FegwvBR/2UYWDbe2bYfhMUYR9vDoJCXP8vSQfk7wKPiG7KrS1tW9GSZOPSV5FkZDdq5OQsHH8veSeMS+wce5TLPj5lcTe3dzrySWxQYOIedYgSVYPJi4FOhRSgTaL7RsOFcvXMXntETx75ghOr9ja60mKOxPNbE+0Z8XbcGAEJAtUIxAJnZjj7wOpSV7nPXA9NTet7dU16hcez4q5t6dfx0UZX1bhl4nBogoHD4ZthREhUpO8blhUw7uzl/bqGtNXLGDSre+2H0gqiaYmnywMgQjoxBx/H0hN8kr2733rIzlAmVw+0EerQkQVtRCP4cWd5BVrGd/rS8RahMSOnT4aFTIR0Ik5fh9IVCQpG35kt2WSH+1Jh4O84Z3kgCIa/ZJ6aJUlIjILJ6F4HLhLVRd3Ot8fuB/4NLAL+HtVbfDNXiMw4i1CXWtLt2VGxBPpcJA3vBNvKeB4fiYioBNz/D5Qe/4SGs/pvnLOefC6dDjIG95xVuUskvVNVNHW7LqwIhIHfgR8FtgC1IrIClXd4Cl2FbBbVSeIyFzge8Df+2y1EQDjF9Vx45LZ3ZbZeMvYdDjIG94Zt7eOImoeRUIn5vh9oCpeSVW8+zIzTn+DNXdMB2Dm9NeLJ7zjQVXRtqxjl1OBelXdBCAiDwFzAG+FngPc7O4/CvxQRERVC/jJXmmSbG6GHgZAjF41lgnN1wAwak2yuMI7HqKgE3P8AXHPmBdgzAthm5FX9rD7qafbHvYOsB4gIq94Xi9V1dQTvpHAZs+5LcC0TpdMl1HVNhH5KzAUKOAne0ZXVCxfx/jlYVuRfzrppDuNQJ50EpjjF5HBwF3A8TgDmq4E3gIeBsYCDcDFqro7KJsMf1HVWWHbUOiYToqfKOgkyAlcdwKrVHUS8AlgI3Aj8HtVnQj83n1tlAbvA6M9r0e5xzKWEZEy4DCch1fFjOnE8JIXnQTi+EXkMOBUYBmAqraq6oc4san73GL3ARcEYY8RCWqBiSJSIyLlwFxgRacyK4DL3P0vAM8Wc3zfdGJkIC86CarFXwM0AveIyH+LyF0iUgkMU9VtbpntwLCA7DFCRlXbgGuBp3BatY+o6noRuVVEUsM/lgFDRaQeWEDxt3RNJ0YH8qWToGL8ZcCngC+r6ssiciedjFNVFcm8VoGIzAfmA4wZac+jiwVVfRJ4stOx73j2W4C/C9quEOm1TrwaGUDxjRgrZfKhk6Ba/FuALar6svv6UZwKvkNEhgO4fzOO31LVpao6RVWnVA/tYdykYRQuvdaJVyP96B+YwUZhEojjV9XtwGYROcY9dAbOOFRvbOoy4Ikg7DGMKGI6MYIiyLjJl4EH3AcUm4ArcP7xPCIiVwHvARcHaI9hRBHTiZF3AnP8qvoaMCXDqTOCssEwoo7pxAgCKbTRcSLSCDQTvdmbVUTPJvDfrqNUtdrH6xk+E2GNQDR1UnIaKTjHDyAir6hqplZRaETRJoiuXUZ+iervHkW7omhTvrHUi4ZhGCWGOX7DMIwSo1Adf+9yuOWXKNoE0bXLyC9R/d2jaFcUbcorBRnjNwzDMHpPobb4DcMwjF5ijt8wDKPEKCjHLyKzROQtEakXkVBWahSR0SLynIhsEJH1IvJV9/jNIvK+iLzmbueEYFuDiLzu3v8V99gQEXlaRN52/x4etF1GsJhOerSt5HVSMDF+N+nwn/EkHQbmdUo6HIQdw4HhqvonERkEvIqzPvrFwF5VvS1IezrZ1gBMUdUmz7H/AD5Q1cWuEzhcVW8Iy0Yjv5hOsrKtgRLXSSG1+NNJh1W1FUglHQ4UVd2mqn9y9/fgrJE9Mmg7csCSeJQWppPeUVI6KSTHnynpcKgVSUTGAp8EUsvoXisidSJyd0hdRQV+JyKvuuuzgyXxKDVMJz1T8jopJMcfKUTkEOBXwNdU9SPgv4DxwInANuD2EMw6WVU/BZwNfElETvWedNOxFUZszygKTCfRpJAcfzZJhwNBRPrhVOYHVPUxAFXdoaoJVU0CP8XpcgeKqr7v/t0JPO7akFWyG6NoMJ30gOmksBx/NkmH846ICE6Oy42q+n3P8eGeYhcCbwRsV6X7EA03T+tZrg2WxKO0MJ10b5fphGATsfQJVW0TkVTS4Thwt6quD8GUGcClwOsi8pp77CZgnoiciNNFbACuDtiuYcDjjt4oAx5U1VUiUosl8SgZTCc9YjqhgIZzGoZhGP5QSKEewzAMwwfM8RuGYZQY5vgNwzBKDHP8hmEYJYY5fsMwjBLDHL9hGEaJYY7fMAyjxPj/rCTxn7RO6YIAAAAASUVORK5CYII=", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEVCAYAAADn6Y5lAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAzrklEQVR4nO3de5wU9Znv8c93mvuIF5wREVCuKppNTJb1skTDeguigpqsq7tmveWge2JumsTbJsdoNsfs0SA5m7MJES9Z72sUiRpNjBpjgoq6Lop4QcSAIDcxwsAwTPdz/qiaoWbsmZ5LdVX19PN+vepFd1d1zzPT9Tz86ldVv5/MDOecc9WjJu0AnHPOJcsLv3POVRkv/M45V2W88DvnXJXxwu+cc1XGC79zzlUZL/xdJGmJpKlpx+FcJZG0QtKxXdjOJE3o4c/o8XurVb+0A6gUZnZw2jE451wcvMXvnHNVxgt/F7Ucskq6StJ/SrpN0mZJL0vaX9LlktZJWinp+Mj7zpW0NNx2uaQL2n3utyStkbRa0hejh62SBkq6TtKfJK2V9BNJg5P+3Z3rLUmHSloo6YNwf/83SQPabTY9zJENkv6PpJrI+88L82iTpEcl7Zfwr9CneOHvmZOB/wD2AP4LeJTgbzkSuBr4aWTbdcBJwK7AucBsSZ8CkDQNuBg4FpgATG33c64F9gcOCdePBL5Tht/HuXLLA18H6oAjgGOA/9lum1OBycCngJnAeQCSZgJXAKcB9cDvgTsTibqPko/V0zWSVgBfBD4NTDGz48LXTybYCXczs7ykocCHwB5m9kGRz5kPPGFmcyTdBKw1s8vDdROAN4GJwFvAFuDjZvZWuP4I4A4zG1vO39W5uLTkjZk91u71rwGfMbNTw+cGnGBmj4TP/yfwOTM7RtKvgHvNbF64roYgNyaZ2Tvheyea2bKkfq9K5y3+nlkbebwN2GBm+chzgF0AJJ0g6RlJ70v6AJhO0OoB2AdYGfms6ON6YAjwQnh4/AHwSPi6cxUl7A59UNJ7kj4Evs/OPGgR3f/fIcgPgP2AOZE8eB8QwRGw6wEv/GUkaSDwC+A6YLiZ7Q48TLDTAqwBRkXeMjryeAPBfyIHm9nu4bKbme1S/sidi92/A68RtMx3Jei6Ubttovv/vsDq8PFK4IJIHuxuZoPN7I9lj7qP8sJfXgOAgcB6oFnSCcDxkfX3AOdKmiRpCPDtlhVmVgB+RnBOYC8ASSMlfTax6J2LT0sX6BZJBwL/VGSbb0raQ9Jo4KvA3eHrPwEul3QwgKTdJP1tEkH3VV74y8jMNgNfISjwm4C/BxZE1v8K+BHwBLAMeCZctT3899KW18PD48eAAxIJ3rl4fYNg/99M0KC5u8g2DwAvAC8BDwHzAMzsfuAHwF1hHrwCnFD+kPsuP7mbIZImEezUA82sOe14nHN9k7f4Uybp1PB6/T0IWjW/9KLvnCsnL/zpu4DgWv+3CK51Ltb36ZxzsUm98EuaJul1ScskXZZ2PEkzs2nh1TrDzOxUM1uTdkwue6o9T1y8Uu3jl5QD3gCOA1YBi4AzzezV1IJyLmM8T1zc0h6d81BgmZktB5B0F8Gt2h3u0AM00AZRm1B4rr3NbNpgZn4TWbK6lSeeI+mqhBxJu/CPpO3dequAw9pvJGkWMAtgEEM4TMckE537iMfs3nfSjqEKlcwTz5HsqIQcSb2PvyvMbK6ZTTazyf0ZmHY4zmWO54jrjrQL/7u0vU17VPiac24nzxMXq7QL/yJgoqSx4djcZxC5s9U5B3ieuJil2sdvZs2SLiIYzz4H3GRmS9KMybms8TxxcUv75C5m9jDBiJXOuQ54nrg4pd3V45xzrhOSbgqndX2lg/WS9KPw5r7FLTP8dcYLv3POZdstwLRO1p9AMGvfRIJLev+91Ad64XfOuQwzs6cIZh3ryEzg5xZ4Bthd0ojOPjP1Pv5qtPW0w1h91M7Jh3KNYvw1iyk0NKQYVe999m+G2Ib3C63PX1y8/VEz66yl4lxRfTVHoG2evLh4+xKgMbJ6rpnN7eZHFrvBbyTBDH9FeeFPSM3QodTsEtxGv+p44+0ZP21dt7RpKxffdh79Nn6AFQrk162HCpwnYd37zfzuV3u3Pt9t5Mr2c6o616FqyBFomye7jVzZaGaTk47BC39Cll35MRaccT0A9TmDyFgq+/cfxJyH5pE38WrT3sybchj59etTirTnDNhOvuR2zhVTDTkCZcmTbt/g5338ZVZTW8vya4/gqKkvM2nAECYNGEJdru0AWjnVsH//WiYNGMJnBq9h6TVj2XbKoSlF3HMFjEYrtC7OdUU15Qi0zZOYLAD+Mby653Dgz6WGd/cWfxnVDB0KY0fy0JnXsX//ro2WWJer5e0ZcxlbmMWkhXtV1CFtwcTWgkpv6Fyo2nIEup8nku4EpgJ1klYB/wvoD2BmPyG4v2M6wfzcW4FzS32mF/4yWnblx3jozOsY329wt9+76OTZ/O7YERV1SGtAo+XSDsNVkGrLEeh+npjZmSXWG/Cl7sTghb+MCoOsy62Y9upytRw4YC3UVE4LuoBoNN+lXNdVW45ANvLEs7QcJPoN34vCoN714dVgWP0warY0VMRlbAVEgw1IOwxXCao0RyAbeeInd8sgt1c95z21kEUnzu7V57RcyfDWlR+PKbLyKphotP6ti3MdqdYcgbZ5khZv8ZeBJA4a8N5HrkzorpYrGQoDK+PElSEaC17wXWnVmiOQjTzxwu9ik6eGhoLP/uRcZ7KQJ174XWzMxHbv4nGuU1nIE+/jL4PC5i3MvOMSzv3Tkb36nA35BsbdfwH7PFUZN0MVwkPYlsW5jlRrjkDbPEmLF/4yKDQ0MPaKhTz1zMG9+py1+RoOvPptBj/wXEyRlVcBsbUwsHVxriPVmiPQNk/S4l09LjZm6Z+0ci7rspAn3uIvo5FPFhg7fxYb8t2/vvisFVM57faLsS2VcW0yBC2Z7YV+rYtzpVRbjkDbPEmLF/4yGjz/OSZ9dwWPb9unyzt23gosadrGoicmMebKhRVzUwq0jEEyoHVxrpRqyxFomydp8cJfZvl167llyqH81UNf79L2b+xo5JsnnsO47y8uc2TxM2/xux6ophyBtnmSlkR+sqTRwM+B4QRjFM01szmShgF3A2OAFcDpZrYpiZgSY0Z+/XpGPzKWCQ0XAnDklCXcvO/vWzfZkG/g0AUXU9Moco1i3IrKnGnIEE1e8HusavOkinIEspEnSf30ZuASM3tR0lDgBUm/Ac4Bfmtm10q6DLgMuDShmBI1eP5zjJ8fPP7DtUew+IzftK57dfsoDrxqeesIg5VzYVpbBRPb8n5ytxeqOk+qIUcgG3mSSOEPJwVYEz7eLGkpwZyQMwnGmQa4FXiSPrhDtzf+msVcNnvGzhcKRn7DhvQCiomZaMp7i7+nPE926qs5AtnIk8R/uqQxwCeBZ4HhkZli3iM4xC32nlnALIBBDEkgyo7VDBnCO18/pMOxQUb8sZkBjyzq9DMKDQ1QoYepnSkATQUfjz8O3c2TTOVIbS3Lvv1xCoOK58jIJwsMnt/5dfd9NUcgG3mSaOGXtAvwC+BrZvahtHMcbTMzSUX3lHDW+bkAu2pYaqMx1dTWon33YdbfP8zEge8V3ebr/c9lwgv1QeukgmYFioOZaPQWf6/1JE8ykyNdmFFr7IDKnDkrLlnIk8Su6pHUn2Bnvt3M7gtfXitpRLh+BLAuqXh64p2vfYLj713EuAEdh3nFqb9g3MMfkhu2R4KRZUMB0ZTPtS6u+yo9T5Zd+THm/HJepzNqLTp5Nuf//hlydXUJRpYd0TxJSyKFX0GTZR6w1Mx+GFm1ADg7fHw28EAS8XRXzZAhrLzyr9nt02uZOPA9cur41FJ9vw+ZPPRt3vzmATRN+6sEo0yfGewo5FoX1z2VnCfRCdP3719LTh2XlrpcbcVPmN4b0TxJS1It/inAF4CjJb0ULtOBa4HjJL0JHBs+zxzV1vLFMx/hnyc+1KXt6/t9yA2fv5n3DquuK1wM0dTcr3Vx3VaxeVIzdBcWnHE98/Z9ukvbt0yYvvrI6ruVKJonaUnqqp6ngY4mxjwmiRhc+ZlBc6H6EjkunifVIQt54lnqYiSa8zWtS8mtpWmSXpe0LLw+vf36cyStj7R+v1iWsJ1LlLqcI+Xix+MuNmawo7lr/ZaScsCPgeOAVcAiSQvM7NV2m95tZhfFG6lz6elOnpSLt/hdbMxEvlDTupRwKLDMzJabWRNwF8GNSs71adE8SYsXfherQl6tC1An6fnIMiuy6UhgZeT5qvC19j4nabGke8OxbJyreJEcSYV39XSFFVi2bS+G9dtCfb8PS26etxqWbd+b3PYEYssQMyg0t2lLbDCzyb34yF8Cd5rZdkkXEAxXcHRvYnTlYWa83DSC+txq6nLFb9yKyluB13ZsJ9eYXvFLS5E8SZy3+Lsgv2EjK6YN4V/mf65L2y/bvjePnfZJRv/opfIGljUmCvma1qWEd4FoC35U+NrOjzPbaGYt/33eCPxlbLG6WFXb0Mq9EsmTtHjh76L8xvcZ9WQzX//Pc1nfvGuH21312gzm3T4NW7WGwtatCUaYEQXtXDq3CJgoaaykAcAZBDcqtWq5WzU0A1gaa6wuPq1DK1NyRq2zVkzllNsvwVasqtihlXutazlSNt7V0w0DHlnEhOf35LljxzF28Pqi22x7uo5R1/6xooeN7TEDa+7azmxmzZIuAh4FcsBNZrZE0tXA82a2APiKpBkEwxW/TzA8scuwwfOfY9LCvXj82H04cMDaotu0zpyVcGyZYWAp9u+DF/5uC7p9hrFCY4quH731pareodWNHdrMHgYebvfadyKPLwcujy0+l4iWbh9qiu8L47Ysrt4cgSBPuthAguB+F2AOQQPpRjO7tt36c4D/w86u0n8zsxs7+0wv/D2Q3/h+2iFklCDllozLgLDbx3Wk63lSrvtdvI/fxSdsybQszrkiwiPjLh4dl+V+Fy/8LlYqqHVxzhUXKfyd3esCZbrfxbt6XHwMlE87COcyrm2e9PZeF+jB/S7e4nexUbhDtyzOuY+K5kkXlOV+Fy/8Llbex+9cad3IkbLc7+JdPS4+3tXjXGndyJNy3e/ihd/FqpNZKZ1zoe7kSTnud/HC72Ijg5rmtKNwLtuykCde+EvYdsqhHc4LmmsU476/uHrHG2nPu3qq0tbTDmP1UcX7q3ONYvw1niNtZCBPvPAXUVNbS83QXQBYeTy8NGN20e2WNg3g6jv+nn4bP8DMyK9bH4y5WsXS3qFdMmqGDqVml2D45VXHG2/P+GnR7ZY2beXi284LcqRQ8BwJpZ0nXviLeOvKj/PzM/4NgPrcNoJzKh91QP/tXPXgbRSshiXbR/KLo/6ium9Vz8AhrEvGsis/xoIzrgegPmdA8TH49+8/iDkPzSNv4tWmvZk35bDqzhHIRJ4kWvjDcSeeB941s5MkjSW4BXlP4AXgC+FtyakqDDQmDWgJo+O5MWskDugffIN5VkPNxxOILrtkfnK3tyomRwYZkwYMKbldTjXs3z/4T6HA2g4HbqsmWciTpK/j/yptrzH9ATDbzCYAm4DzE46nLYnc8L3ID+7+t5LDoG4PampLzz7Ul/kNXL2W+Rzpt/dwCoO6nyM1GFY/rOpzBNLPkcQKv6RRwIkEd5YhSQS3Fd8bbnIrcEpS8RSTq6vj9N+9xGMn/rDb723p9ll+WRW3+sND2JbFdU9F5Mhe9Zz31EIWnVj8vFdnWrp93rqyinME2uRJWpJs8d8AfAtah+LeE/jAzFp+/Y4GH0LSrJZBjHZQxolsa8SBA9ZQl+u4e6fjtwbdPoWBZYirUviQDb11AxnPEUkcNOC9Ls2r215Lt09hYJWf3O3ekA1lkUgfv6STgHVm9oKkqd19v5nNBeYC7Kphmd5r8oMK5IbvVXxlwchv2NBnr2oQXvB7qqpyZHCBfiP2LrquGq78yUKeJHVydwowQ9J0YBCwK8GMMrtL6he2aD4y+FAluu/kH/HWtPqi615vHMHTR48iv2FjwlElxKCmue8mbJlVTY4sOnk266cXP8lbFVf+ZCBPEin80VuKw9bMN8zsHyT9J/B5gqsWzgYeSCKecqrL7aC/is81umduC/Ou/BtGPzaOgQ8tSjiyBBjUeIu/R6orR2qp66A3tT63hm9cM5bRj4xl8Pznkg0sKRnIk7RH57wUuFjSMoL+zHkpx1NW9bkGFpw6m1VH58jV14P61qVtwSGstS4uFlWVI3W5Wt6eMZeVxxN0mfaxHIG2eZKWxG/gMrMngSfDx8sJpharKjed+hOePu6AvtftY1CzI+0gKp/nSNAd9LtjR/TNbp8M5EnaLf5MsS0NnH37RfyPFSf16P2bC3m2d+E/8T1rtvHXtW/y+pUT2X7iX/XoZ2WSQU3eWhfX9xQ2b2HmHZdw7p+OLOvPqcvV8pnBa1h6zVi2ndLH/t+L5ElavPBHFBoaGPPthbz47MTuv9eMzYUaGq1rl4K2dPusntJ3Rs2QmXf19HGFhgbGXrGQp545uOw/q6Xbp6NBEitVNE/S0neqjkufQc0OL/jOdSoDeeKFv4h9niowOX8xAJ/86zf42ZgHO9x2cyE4PR/ccdO3WiY94S396jDyyQIT8hcCcOSUJdy87+9TjqiypJ0nXviLGPzAc4wLL5r77385gv/eZ3DrugHkmdC/kRzB1QabCzUU6HtXHvSEwr5L1/cNnv8c4+cHj/9w7REsPuM3rev6U2D//oPIyRtCxWQhT7zwlzDu+4u59kfH73xhz9258IEHmdh/Q3pBZZUZ2uHDc1ab8dcs5rLZM3a+ULcHNzx8U+uonK6dDOSJF/4SCg0NEJk9KAfssH7eyi8mA3ckuuS1z5F+Ennz/OhQBvLEC7+LlfLe4neulLTzxAu/i43M0A4fs8G5zmQhT7zwu/gYqNlb/M51KgN54qfdu6tgLG3ch/X53p24yiNeaxpOv219qC/UDPL5nYurSmbGy00j2JBvKL1xJ/JWYEnTNnKNfShHoG2epMQLfzflN2zgj0eP5PwFs3r1OW827cW8mZ9lv+tfiiewLDBQU751cdUpv249t0w5lL966Ou9+pw3djTyzRPPYdz3F8cUWUZE8iQtXvi7y4z8ho2MfizPzF98nY2FwaXf085Xlv0d3/6Ps7B33qWwdWsZgkyLt/gdQY6sX8/oR2Ds/Fk9avmftWIqp9x+CbZiVXDVUJ/iLf6KNfChRex/7Vs8teXALnf7tHTvrPndKEZf88c+VvQJDmGbm3curqoNnv8ck767gse37dPl4t/SvbPoiUmMuXJhHyz6tM2TlHjh74Xudvv0ye6dKDPY0bxzcVWvu90+fbZ7JyqaJynxq3p6o7XbZxynbru45Ob9ton93nmp77X0Wxje0ndttXb7jGVCw4UlN881inErFvfNln6LDOSJF/4YDHxoEWMf6tq2ffpiRzPMW/quiOjYPqX06RyBTOSJd/W4+JhhTU2tSymSpkl6XdIySZcVWT9Q0t3h+mcljSlH2M4lKpInXVGOPPHC7+LTjZO7knLAj4ETgIOAMyUd1G6z84FNZjYBmA38oAxRO5esbpzcLVeeeOF3sTEzCjuaW5cSDgWWmdlyM2sC7gJmtttmJnBr+Phe4BipD86+7apKNE+6oCx54n38Ljab2fToY4V76iIvDZL0fOT5XDObGz4eCayMrFsFHNbuI1u3MbNmSX8G9gR8TGxXsdrlSWc5AmXKk8QKv6TdgRuBjxGc1z4PeB24GxgDrABON7NNScXk4mVm09KOodJ5nvR9WciTJLt65gCPmNmBwCeApcBlwG/NbCLw2/C5qw7vAqMjz0eFrxXdRlI/YDdgYyLRpcfzxEWVJU8SKfySdgOOAuYBmFmTmX1A276pW4FTkojHZcIiYKKksZIGAGcAC9ptswA4O3z8eeBxM+uzM714nrgiypInSbX4xwLrgZsl/ZekGyXVAsPNbE24zXvA8GJvljRL0vOSnt/B9oRCduVkZs3ARcCjBK3ae8xsiaSrJbXM4zcP2FPSMuBi+n5Lt8d54jnSN5UrT5REA0rSZOAZYIqZPStpDvAh8GUz2z2y3SYz26Ozz9pVw+wwHVPWeF3HHrN7XzCzyWnH0RfFlSeeI+mqhBxJqsW/ClhlZs+Gz+8FPgWslTQCIPx3XULxOJdFnicuEYkUfjN7D1gp6YDwpWOAV2nbN3U28EAS8TiXRZ4nLilJXsf/ZeD28ATFcuBcgv947pF0PvAOcHqC8TiXRZ4nruwS6eOPk6T1QAPZu4mnjuzFBPHHtZ+Z1cf4eS5mGc4RyGaeVF2OVFzhB5D0fNZOnmQxJshuXK68svq9ZzGuLMZUbj5Wj3POVRkv/M45V2UqtfDPLb1J4rIYE2Q3LldeWf3esxhXFmMqq4rs43fOOddzldrid84510Ne+J1zrspUVOEvNfdkQjGMlvSEpFclLZH01fD1qyS9K+mlcJmeQmwrJL0c/vznw9eGSfqNpDfDfzsdC8lVPs+TkrFVfZ5UTB9/OPfkG8BxBGOaLALONLNXE45jBDDCzF6UNBR4gWCY3NOBLWZ2XZLxtIttBTDZzDZEXvtX4H0zuzYsAnuY2aVpxejKy/OkS7GtoMrzpJJa/F2Ze7LszGyNmb0YPt5MMFTqyKTj6AYfy726eJ70TFXlSSUV/mJzT6a6I0kaA3wSaBlN8SJJiyXdlNKhogG/lvSCpFnha12a88D1GZ4npVV9nlRS4c8USbsAvwC+ZmYfAv8OjAcOAdYA16cQ1qfN7FPACcCXJB0VXRnOylMZfXuuT/A8yaZKKvxdmXsyEZL6E+zMt5vZfQBmttbM8mZWAH5GcMidKDN7N/x3HXB/GIOP5V5dPE9K8DyprMLflbkny06SCKY6W2pmP4y8PiKy2anAKwnHVRueRCOcru/4MAYfy726eJ50HpfnCcmOx98rZtYsqWXuyRxwk5ktSSGUKcAXgJclvRS+dgVwpqRDCA4RVwAXJBzXcOD+IN/oB9xhZo9IWoSP5V41PE9K8jyhgi7ndM45F49K6upxzjkXAy/8zjlXZbzwO+dclfHC75xzVcYLv3POVRkv/M45V2W88DvnXJUpWfjDsauPjfOHSpoqaVWcn5kESWMkmaRu3fgm6RxJT5crrqRF9wlJV0i6sYefs0TS1DhjS5ukA8Jx3jdL+kra8STFcyNQKblRMXfuumwys+93ZTtJtwCrzOyfI+89uFxxpehbwBNmdkjagbh0ZTk3Kr6rJ5x4wvVQd1torqT9gB4NkRD3d+G50Tt9OjfMrNOFYDyNy4FXgU3AzcCgcN0ewIPA+nDdg8CoyHuHhduvDtfPD1+fSvA/XMt2Xwk/f1T4/FsEQ7auBr5IMK7HhHDdLQRDuz4MNADHApOAJ4EPCJJuRuSznwS+GHl+DvB05LkBFwJvhu//MTuHssgB1wEbgOXAl8Lt+3XwtxoN3Bf+PTYC/9bBz5xDMGb6hwQzEx0ZWXco8Hy4bi3ww/D1QcBt4ed+QDAY1/AefGdTCcZov5Rg3PH/IGgAXAa8FX7+PcCwyOd9gWD8ko3AleHnHxuuuwq4LbLtp4E/hjGuDH/3WcAOoAnYAvwyEmfL5wwEbgi/89Xh44HtYr6EYNTENcC5pfbdpBfgcSAPNIa/5/7AbsDPw33iHeCfgZrIfvEHYHb4t/1e+He4DvhT+P3/BBgc+RmeG54bN9DL3Ohq4X8l/OKGEeyo3wvX7Ql8DhgCDAX+k7C4h+sfAu4m+A+iP/CZaLDh4+8ALwL14fNp4R/94PBzb+OjO/efCQaBqgl/7jKCAaAGAEcDm4EDurFzPwjsDuxLsGNOC9ddCLwW+d2foIOdmyAR/psgiWsJdsZPd/Azzwr/dv3CL+w9du58C4EvhI93AQ4PH18A/DL8m+SAvwR27cF3NhVoBn5AsEMNBr4KPEMwhO9A4KfAneH2BxHskEeF634Yvv8jOzdBa3czcGb4fe8JHBL53r5XJM6Wz7k6jGEvoJ4gQa5pF/PV4edOB7YSTI+XesFv9zu1399+TjDS41BgDMG0iOdH9otm4MvhvjA43H8WhN/b0PA7/9+eG54bxJgbXS38F0aeTwfe6mDbQ4BN4eMRQKFYAGGw74Z/qKeB3SLrbiLc0cPnE/jozv3zyPojw52jJvLancBV3di5Px15fg9wWfj48Xa/+/F0vHMfQZAYxda1+ZlF1m8CPhE+fgr4LlDXbpvzwi/84735zsK/fRNhMoWvLQWOiTwfQdAK6UfwH/NdkXW14fuL7dyXA/d3ENMtdL5zvwVMj6z7LLAiEvO26N+WoHVzeKm/RdJLdH8jKEJNwEGR9RcAT0b2iz9F1omgpT6+3X71tueG50acudHVPv7oVG7vAPsASBoi6aeS3pH0YfjF7B72LY4mmLx4UwefuTvBYc7/NrM/R17fp93Piz4u9to+wEoLJnaIxtid6ebeizzeStCaKBbLO518xmjgHTNrLvXDJH1D0lJJf5b0AUF3QF24+nyCLoLXJC2SdFL4+n8QDLV7l6TVkv41nOiiI0W/s9B6M2uMPN+PYKjaD8J4lhJ0WQyn3d/AzBoIDmuLGU2wk/bEPrT9+7aPeWO7v230e8qqOoJWWPvfK7pvRr+neoJW6wuR7+KR8HXw3PDcKB5zt3Ojq4U/OqPPvgT9TBAcih0AHGZmuxIc8kDQclkJDJO0ewefuQk4CbhZ0pTI62sIDquK/ewWFnm8GhgtKfq77MvOWYcaCJKpxd4dxFPMGj76u3dkJbBvqRNCko4k6Kc9neBoaHeCw3MBmNmbZnYmwWHdD4B7JdWa2Q4z+66ZHQT8NcHf7h87+VEdfWfw0WnlVgInmNnukWWQBTMVtfkbSBpCcJhazEqCafWKaf8z21tNkGQdxVyJNhC0Dtv/XtEZsazd9tuAgyPfw25m1pLEnhueG8Vi7rauFv4vSRolaRjBCYy7w9eHEuyoH4Tr/lfLGyyYuPhXwP+TtIek/kXmtnwS+AfgPkktU7DdA5wraVL4h/x2idieJfgf7lvhz5gKnAzcFa5/CTgtPDqZQNBq6Kp7gK+Ev/seBCd5OvIcwY5wbTjLz6B2/6G1GErQJ7ce6CfpO8CuLSslnSWpPmylfRC+XJD0N5L+Ijya+pCgoBToWEffWTE/Af5F0n5hDPWSZobr7gVOkvRpBTM6XU3H+83twLGSTpfUT9Ke4aQbEJyMG9dJDHcC/xz+7DqCw+jbOtk+88wsT7AP/YukoeHf92I6+L1s53SEsyXtBSBppKTPhpt4bnhuxJIbXS38dwC/Jjh7/xbB1QcQnF0eTNBSeYbgsDTqCwRfwmsE/U5fa//BZvYbgj66X0r6lJn9CvgRwcmiZeHnAmwvFpiZNRHszCeEcfw/4B/N7LVwk9kE/W5rgVsJvoCu+hnBIeR/E5yAvq+jDcMkP5mg3/VPBGfa/67Ipo8S/J3eIDhka6Ttoec0YImkLQRXOJxhZtsIWmP3EuzYS4HfERzidqSj76yYOQQnFH8taTPB3/yw8PdaQnDFxh0Eybsp/N2K/Q3+RNBnegnwPkFh+US4eh5wUHjIPL/I279HcMXGYuBlgr93ZzFXii8TtKyXE5zPuoOgr74jlxLu92H36WMER9V4bnhuEFNuZH4GLkmTCM7CD+xKH6EL7h4kOGn3WNqxuPLx3Og+z41AJm/gknSqpIHhIeQPCK5t9R3bVT3PDReH1Au/pGmSXpe0TFJLP+EFBF1DbxGcQf+n1AJ0LgNa8oSgb/fPeG64Xki1qyc8GfMGcBxB39gi4EwzezW1oJzLGM8TF7e0W/yHAsvMbHl4IuouYGaJ9zhXbTxPXKzSHoRoJG3P2q8iPGPekbphORszurN7M1w5vbB4+wYzqy+9pYtRt/JkgAbaIGrLHpQrbjObMp8jaRf+LpE0i+AuX/Yd2Y/nHi1234pLQm7Ess7u0HQpiebIIIZwmI5JOaLq9Zjdm/kcSbur513a3kU3irZ3NQJgZnPNbLKZTa7f00eadVWnZJ5Ec6Q/AxMNzlWetAv/ImCipLHhnW9nENws4ZzbyfPExSrVrh4za5Z0EcEdezngpvBuOOdcyPPExS31Pn4ze5hg4gjnXAc8T1yc0u7qcVVM0k2S1kl6pYP1kvSj8Oa+xZI+lXSMzqWtHHnihd+l6RaCgbc6cgIwMVxmEUwr6Fy1uYWY8yT1rp5qtC7fwHv54lcn9afA/v0HkVPf/z/ZzJ6SNKaTTWYSzChlBKNV7i5pRDjkt+vDaoYOpWaX4vciWKFAft16yPgAk3EpR5544U/B4Qsu5sCr3y6+sm4Pbnj4JvbvX3k34Hz2b2pt4/v51ucvLN6+hGBo3RZzzWxuNz6y2I1LIwmGwHV92LIrP8aCM64vuu7Vpr2ZN+Uw8uvXJxxVPKJ5EkOOQA/yxAt/Qs5aMZVnFh4IwKinCuTXriu6Xc2WBk66/RtMOfoVbt73963v/cNL+7Po5NnU5bL7H8L695v5wyM7Z4Qbss+KRjObnGJIroJsPe0wVh8lAKYe/jKTBgwpul19bg3fuGYsox8Zy+D5z7W+d+2hNYy/ZjGFhobEYu6JaJ6klSNe+MssbwVe27GdRU9MYvyVC0tuX2hoYOwVC/nDtUew+IzfALDoiUkc9H+Xs366qMvw/WuGsT3eEYK7dIOfq3AS/Ybvxarjjbdn/LTk5nW5Wt6eMZexhVkc9GwwW+Sq440Hp/2Qb86eCRkv/FnIEy/8ZfbGjka+eeK5jFuxuNO54Nobf81iLps9A4BxWxbD0KzPKx7MdbfV8iW364YFwEWS7iIYm+bP3r/f9+T2que8pxbymcH3QTfGGFp08mzWTw+OEOpzxtp8ZZwXy0KeeOEvo7NWTGXRE5OCot/NVkihoeEjLZeZd1zSpgsoawpmNHbjhJukO4GpQJ2kVQRzNvcHMLOfEFy3Pp1gmsGtwLkxh+xStvW0w1h1vPGZwfd3uxuzLlfb7gi4gaXfHcPoR8a0dgFlURbyxAt/GUS7d8ZcubBbLf2OtHQBPXXD4ZDRwm+IRut6q8vMziyx3gjmNHV9TZvunbl0p6XfkZYuoAkNFzJ+fq8/rmyykCde+Mugp907lS6PaDAfMtuV1tPunb4gC3nihT9mveneqXSGaCz4LuU615vunb4gC3niWRqzZxYeyPiYuncqTcFEo7f4XQmrj1J49U71FX3IRp544XexKSC2mo8F71xnspAnXvhdbIJDWG/xO9eZLORJZVz4WkHygwvkhu8FUtqhJK5AcAjbsjhXTK5RLG3aSt6qsUO0bZ6kxQt/zBadPJvzf/8Mubq6tENJXMFEQ2Fg6+JcMeOvWcwlJ57LW83b0g4lFdE8SYt39cSsLlfLgQPWQk11tvi3e1ePK6HQ0EC/jR+Qt+rLEchGnnjhd7EJbkzxXcq5zmQhTzxLXWwKVsPWvHfxONeZLOSJF34Xm+AQ1ncp5zqThTzxLC2DGgyrH0bNloZY7t6NzkZUGJTdKyHMSH2HdpXBzHi5aQT1udWx3L0bndUu15jtcwdZyJNEfrqk0cDPgeGAEcwyM0fSMOBuYAywAjjdzDYlEVM57d9/EHMemsdJt3+DsVeUHoO/lOhsRPU5I6t3PBaoYVt+QNphVKxqypP8uvXcMuVQLr1mbDhIW+9EZ7UbtyXbY2RlIU+SupyzGbjEzA4CDge+JOkg4DLgt2Y2Efht+Lzi5VTD/v1rmXL0K7x97RHU1AaFuqa2luXXHsG2Uw5t87iYracdxrIbDmfZDYdz1NRgNqJJA4ZkemyTlpZMy+K6rXryxIz8+vWMfgTGzp/FhnxwZLwh38C4+y/g3D8d2eZxMWetmMqEOy9kwp0XMurXRn7tOvJr12V+jKxonqQlkZ8cTgqwJny8WdJSgjkhZxKMMw1wK/AkcGkSMSXh5n1/z9IzHuXi286j5u2V1AzdhQVnXM/0Xb/GgctH89CZ1zG9/yWMfyAYojZ601dXZyPKkgKiqZDhKcIyrhrzZPD855i0cC8eP3Yfjh68mrX5Gg68+m3+8PWP8dipr3DgVct56sqDyY/+Ha/t2E6enTnS1VntsiYLeZL4fznhbPGfBJ4FhkdminmP4BC3T4l2+0z80XIguMnr/RNhfL/BwM4hag8a8F7r+7LcpdMRM9GY9+v441BNeRLt9nlw2hwguMnr59dPJr9hAzC+dahzNuzs4cp6l05HspAniRZ+SbsAvwC+ZmYfKtLCNTOTVHRaGkmzgFkA+46srC6EaLfPk8MmUZ+zNjMHHTllCU9+d1KfGKK2gGjKe4u/t3qSJ9EcGUTxScozq7XbZyynbbw4KOiRGehGPlnglIZL+sxQ51nIk8SqqKT+BDvz7WZ2X/jyWkkjzGyNpBHAumLvNbO5wFyAyZ8Y1PU5yzLk5n1/H86cVdul1yuRGakfwla6nuZJNEd21bCKzJHB859jzHw+0orv6PVKlYU8SeTkroImyzxgqZn9MLJqAXB2+Phs4IEk4nHlYYjt+X6ti+sez5PqEM2TtCT1k6cAXwBelvRS+NoVwLXAPZLOB94BTk8oHlcGZmKHd/X0hudJFchCniR1Vc/TQEd3VRyTRAyu/MygOe8DvvaU50l1yEKe+PG4i41l4KSVc1mXhTzxwu9iE7RkvPA715ks5IkXfhcbQ+S9q8e5TmUhT7zwu/gYqe/QzmVeBvLEC7+LjRkUvPA716ks5IkXfhcjUchne0hc59KXfp544XfxMTBv8TvXuQzkiRd+Fx8DvMXvXOcykCde+F2szAu/cyWlnSd+XO7iY0LNO5dSJE2T9LqkZZI+MrmIpHMkrZf0Urh8sSxxO5ck63qOQHnyxFv8Lj7dOISVlAN+DBwHrAIWSVpgZq+22/RuM7so1jidS1MG8sRb/C5WKuxcSjgUWGZmy82sCbiLYKYp5/q8LuYIlClPvPC7+Bjtu3rqJD0fWWZFth4JrIw8XxW+1t7nJC2WdG84GblzlS2SJ3SeI1CmPPGuHhcr5ds83WBmk3vxcb8E7jSz7ZIuIJhv9uhefJ5zmRDJk97mCPQgT7zF72IjA+XVupTwLhBtmYwKX2tlZhvNbHv49EbgL2ML1rmURPOkC8qSJ174XXwMapp3LiUsAiZKGitpAHAGwUxTrcJpBlvMAJbGGa5zqbAu5wiUKU+8q8fFqosnrDCzZkkXAY8COeAmM1si6WrgeTNbAHxF0gygGXgfOKcsQTuXsLTzxAu/i499pI+/883NHgYebvfadyKPLwcujys85zIhA3nihd/FRtblw1fnqlYW8sQLv4tVd1oyzlWrtPPEC7+LTzcPYZ2rShnIk0Sv6pGUk/Rfkh4Mn4+V9Gw4BsXd4VlrV6kM1Lxzcd3nOVIFLP0cSfpyzq/S9lKjHwCzzWwCsAk4P+F4XIwE1BR2Lq5HPEf6uGiepCWxwi9pFHAiwQ0GSBLB3WX3hpvcCpySVDyuDMJD2JbFdY/nSJWw9HMkyRb/DcC3gJb/5/YEPjCzlgOejsagcBWkGzdwuY+6Ac+RqpB2jiRS+CWdBKwzsxd6+P5ZLYMYrd/oTcmskrf4eyzOHNnB9tJvcKlRBlr8SV3VMwWYIWk6MAjYFZgD7C6pX9ii+cgYFC3MbC4wF2DyJwZZMiG7bjNQ3r+eHootR3bVMP8SsiwDeZJIi9/MLjezUWY2hmCsicfN7B+AJ4DPh5udDTyQRDyufLyrp2c8R6pL2jmS9iBtlwIXS1pG0J85L+V4XC/IoCa/c3Gx8BzpY6J5kpbEb+AysyeBJ8PHywlmmHF9QQYOYfsCz5E+LgN54nfuuvgY1Ozwwu9cpzKQJ174XWyEd/E4V0oW8sQLv4uPWeqHsM5lXgbyxAu/i08GDmGdy7wM5IkXfhefDJy0ci7zMpAnXvhdbATUNPvobM51Jgt54oXfxccMeeF3rnMZyBMv/C4+Bmr2rh7nOpWBPPHC7+JjpN6ScS7zMpAnXvhdbGSGdviF/M51Jgt54oXfxchQ3gu/c51LP0+88Lv4ZOAQ1rnMy0CeeOF38TGDHT4es3OdykCeeOF38TGDZi/8znUqA3nihd/Fx4Bm7+N3rlMZyBMv/C4+ZrBjR9pROJdtGcgTL/wuPmaYd/U417kM5EnaUy+6vqTlpFXLUoKkaZJel7RM0mVF1g+UdHe4/llJY8oRtnOJiuZJF5QjT7zwu/iYYU07WpfOSMoBPwZOAA4CzpR0ULvNzgc2mdkEYDbwgzJE7VyyInlSSrnyxAu/i42ZYc07WpcSDgWWmdlyM2sC7gJmtttmJnBr+Phe4BhJijVo5xIWzZMuKEueeB+/i81mNj36m+a76yIvDZL0fOT5XDObGz4eCayMrFsFHNbuI1u3MbNmSX8G9gQ2xBu5c8lplyed5QiUKU8SK/ySdgduBD5GcEHTecDrwN3AGGAFcLqZbUoqJhcvM5uWdgyVzvOk78tCniTZ1TMHeMTMDgQ+ASwFLgN+a2YTgd+Gz111eBcYHXk+Knyt6DaS+gG7ARsTiS49nicuqix5kkjhl7QbcBQwD8DMmszsA9r2Td0KnJJEPC4TFgETJY2VNAA4A1jQbpsFwNnh488Dj5tZnx3w3/PEFVGWPEmqxT8WWA/cLOm/JN0oqRYYbmZrwm3eA4YnFI9LmZk1AxcBjxK0au8xsyWSrpY0I9xsHrCnpGXAxfT9lq7niWujXHmSVB9/P+BTwJfN7FlJc2gXnJmZpKL/S0maBcwC2Hekn4/uK8zsYeDhdq99J/K4EfjbpONKUY/zJJojgxiSRKwuIeXIk6Ra/KuAVWb2bPj8XoIdfK2kEQDhv+uKvdnM5prZZDObXL9nLpGAnUtBj/MkmiP9GZhYwK4yJVL4zew9YKWkA8KXjgFepW3f1NnAA0nE41wWeZ64pCTZb/Jl4PbwBMVy4FyC/3jukXQ+8A5weoLxOJdFnieu7BIr/Gb2EjC5yKpjkorBuazzPHFJUKVdHSdpPdBA9u7erCN7MUH8ce1nZvUxfp6LWYZzBLKZJ1WXIxVX+AEkPW9mxVpFqcliTJDduFx5ZfV7z2JcWYyp3HyQNuecqzJe+J1zrspUauGfW3qTxGUxJshuXK68svq9ZzGuLMZUVhXZx++cc67nKrXF75xzroe88DvnXJWpqMJfatLhhGIYLekJSa9KWiLpq+HrV0l6V9JL4TI9hdhWSHo5/PnPh68Nk/QbSW+G/+6RdFwuWZ4nJWOr+jypmD7+cNLhN4DjCAazWgScaWavJhzHCGCEmb0oaSjwAsH46KcDW8zsuiTjaRfbCmCymW2IvPavwPtmdm1YBPYws0vTitGVl+dJl2JbQZXnSSW1+Lsy6XDZmdkaM3sxfLyZYIzskUnH0Q0+iUd18TzpmarKk0oq/MUmHU51R5I0Bvgk0DKM7kWSFku6KaVDRQN+LemFcHx28Ek8qo3nSWlVnyeVVPgzRdIuwC+Ar5nZh8C/A+OBQ4A1wPUphPVpM/sUcALwJUlHRVeG07FVRt+e6xM8T7Kpkgp/VyYdToSk/gQ78+1mdh+Ama01s7yZFYCfERxyJ8rM3g3/XQfcH8bQpcluXJ/heVKC50llFf6uTDpcdpJEMMflUjP7YeT1EZHNTgVeSTiu2vAkGuE8rceHMfgkHtXF86TzuDxPSHYill4xs2ZJLZMO54CbzGxJCqFMAb4AvCzppfC1K4AzJR1CcIi4Argg4biGA/cH+UY/4A4ze0TSInwSj6rheVKS5wkVdDmnc865eFRSV49zzrkYeOF3zrkq44XfOeeqjBd+55yrMl74nXOuynjhd865KuOF3znnqsz/B1NiAylScTn2AAAAAElFTkSuQmCC", "text/plain": [ "
" ] diff --git a/auto3dseg/notebooks/auto_runner.ipynb b/auto3dseg/notebooks/auto_runner.ipynb index 6af2b8ef15..61c08a2528 100644 --- a/auto3dseg/notebooks/auto_runner.ipynb +++ b/auto3dseg/notebooks/auto_runner.ipynb @@ -46,6 +46,8 @@ "import os\n", "import torch\n", "\n", + "from pathlib import Path\n", + "\n", "from monai.bundle.config_parser import ConfigParser\n", "from monai.apps import download_and_extract\n", "\n", @@ -69,22 +71,29 @@ "name": "stderr", "output_type": "stream", "text": [ - "Task09_Spleen.tar: 1.50GB [00:53, 30.2MB/s] \n" + "Task04_Hippocampus.tar: 27.1MB [00:15, 1.88MB/s] " ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2022-09-28 16:22:52,786 - INFO - Downloaded: Task09_Spleen.tar\n", - "2022-09-28 16:22:52,787 - INFO - Expected md5 is None, skip md5 check for file Task09_Spleen.tar.\n", - "2022-09-28 16:22:52,787 - INFO - Writing into directory: ./.\n" + "2022-10-18 08:11:37,235 - INFO - Downloaded: Task04_Hippocampus.tar\n", + "2022-10-18 08:11:37,235 - INFO - Expected md5 is None, skip md5 check for file Task04_Hippocampus.tar.\n", + "2022-10-18 08:11:37,236 - INFO - Writing into directory: ..\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" ] } ], "source": [ - "root = \"./\"\n", - "msd_task = \"Task09_Spleen\"\n", + "root = str(Path(\".\"))\n", + "msd_task = \"Task04_Hippocampus\"\n", "resource = \"https://msd-for-monai.s3-us-west-2.amazonaws.com/\" + msd_task + \".tar\"\n", "compressed_file = os.path.join(root, msd_task + \".tar\")\n", "if os.path.exists(root):\n", @@ -146,11 +155,11 @@ "name": "stdout", "output_type": "stream", "text": [ - "2022-09-28 16:23:12,874 - INFO - ./work_dir does not exists. Creating...\n", - "2022-09-28 16:23:12,876 - INFO - ./work_dir created to save all results\n", - "2022-09-28 16:23:12,894 - INFO - Loading ./input.yaml for AutoRunner and making a copy in /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/work_dir/input.yaml\n", - "2022-09-28 16:23:12,901 - INFO - The output_dir is not specified. /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/work_dir/ensemble_output will be used to save ensemble predictions\n", - "2022-09-28 16:23:12,904 - INFO - Directory /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/work_dir/ensemble_output is created to save ensemble predictions\n" + "2022-10-18 08:11:37,523 - INFO - ./work_dir does not exists. Creating...\n", + "2022-10-18 08:11:37,524 - INFO - ./work_dir created to save all results\n", + "2022-10-18 08:11:37,524 - INFO - Loading ./input.yaml for AutoRunner and making a copy in /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/work_dir/input.yaml\n", + "2022-10-18 08:11:37,531 - INFO - The output_dir is not specified. /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/work_dir/ensemble_output will be used to save ensemble predictions\n", + "2022-10-18 08:11:37,533 - INFO - Directory /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/work_dir/ensemble_output is created to save ensemble predictions\n" ] } ], @@ -175,8 +184,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "2022-09-28 16:23:13,306 - INFO - Work directory ./work_dir is used to save all results\n", - "2022-09-28 16:23:13,384 - INFO - The output_dir is not specified. /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/work_dir/ensemble_output will be used to save ensemble predictions\n" + "2022-10-18 08:11:37,674 - INFO - Work directory ./work_dir is used to save all results\n", + "2022-10-18 08:11:37,676 - INFO - The output_dir is not specified. /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/work_dir/ensemble_output will be used to save ensemble predictions\n" ] } ], @@ -202,11 +211,11 @@ "name": "stdout", "output_type": "stream", "text": [ - "2022-09-28 16:23:13,659 - INFO - ./my_workspace does not exists. Creating...\n", - "2022-09-28 16:23:13,661 - INFO - ./my_workspace created to save all results\n", - "2022-09-28 16:23:13,677 - INFO - Loading ./input.yaml for AutoRunner and making a copy in /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/my_workspace/input.yaml\n", - "2022-09-28 16:23:13,682 - INFO - The output_dir is not specified. /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/my_workspace/ensemble_output will be used to save ensemble predictions\n", - "2022-09-28 16:23:13,690 - INFO - Directory /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/my_workspace/ensemble_output is created to save ensemble predictions\n" + "2022-10-18 08:11:37,812 - INFO - ./my_workspace does not exists. Creating...\n", + "2022-10-18 08:11:37,813 - INFO - ./my_workspace created to save all results\n", + "2022-10-18 08:11:37,815 - INFO - Loading ./input.yaml for AutoRunner and making a copy in /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/my_workspace/input.yaml\n", + "2022-10-18 08:11:37,818 - INFO - The output_dir is not specified. /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/my_workspace/ensemble_output will be used to save ensemble predictions\n", + "2022-10-18 08:11:37,819 - INFO - Directory /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/my_workspace/ensemble_output is created to save ensemble predictions\n" ] } ], @@ -235,9 +244,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "2022-09-28 16:23:14,063 - INFO - Work directory ./work_dir is used to save all results\n", - "2022-09-28 16:23:14,072 - INFO - Loading ./input.yaml for AutoRunner and making a copy in /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/work_dir/input.yaml\n", - "2022-09-28 16:23:14,085 - INFO - The output_dir is not specified. /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/work_dir/ensemble_output will be used to save ensemble predictions\n" + "2022-10-18 08:11:37,936 - INFO - Work directory ./work_dir is used to save all results\n", + "2022-10-18 08:11:37,938 - INFO - Loading ./input.yaml for AutoRunner and making a copy in /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/work_dir/input.yaml\n", + "2022-10-18 08:11:37,940 - INFO - The output_dir is not specified. /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/work_dir/ensemble_output will be used to save ensemble predictions\n" ] } ], @@ -270,9 +279,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "2022-09-28 16:23:14,446 - INFO - Work directory ./work_dir is used to save all results\n", - "2022-09-28 16:23:14,454 - INFO - Loading ./input.yaml for AutoRunner and making a copy in /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/work_dir/input.yaml\n", - "2022-09-28 16:23:14,475 - INFO - Directory ./output_dir is created to save ensemble predictions\n" + "2022-10-18 08:11:38,055 - INFO - Work directory ./work_dir is used to save all results\n", + "2022-10-18 08:11:38,057 - INFO - Loading ./input.yaml for AutoRunner and making a copy in /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/work_dir/input.yaml\n", + "2022-10-18 08:11:38,061 - INFO - Directory ./output_dir is created to save ensemble predictions\n" ] } ], @@ -298,9 +307,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "2022-09-28 16:23:14,843 - INFO - Work directory ./work_dir is used to save all results\n", - "2022-09-28 16:23:14,853 - INFO - Loading ./input.yaml for AutoRunner and making a copy in /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/work_dir/input.yaml\n", - "2022-09-28 16:23:14,876 - INFO - The output_dir is not specified. /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/work_dir/ensemble_output will be used to save ensemble predictions\n" + "2022-10-18 08:11:38,182 - INFO - Work directory ./work_dir is used to save all results\n", + "2022-10-18 08:11:38,186 - INFO - Loading ./input.yaml for AutoRunner and making a copy in /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/work_dir/input.yaml\n", + "2022-10-18 08:11:38,194 - INFO - The output_dir is not specified. /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/work_dir/ensemble_output will be used to save ensemble predictions\n" ] } ], @@ -326,9 +335,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "2022-09-28 16:23:15,270 - INFO - Work directory ./work_dir is used to save all results\n", - "2022-09-28 16:23:15,390 - INFO - Loading ./input.yaml for AutoRunner and making a copy in /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/work_dir/input.yaml\n", - "2022-09-28 16:23:16,052 - INFO - The output_dir is not specified. /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/work_dir/ensemble_output will be used to save ensemble predictions\n" + "2022-10-18 08:11:38,312 - INFO - Work directory ./work_dir is used to save all results\n", + "2022-10-18 08:11:38,314 - INFO - Loading ./input.yaml for AutoRunner and making a copy in /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/work_dir/input.yaml\n", + "2022-10-18 08:11:38,320 - INFO - The output_dir is not specified. /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/work_dir/ensemble_output will be used to save ensemble predictions\n" ] } ], @@ -395,9 +404,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "2022-09-28 16:23:16,886 - INFO - Work directory ./work_dir is used to save all results\n", - "2022-09-28 16:23:16,899 - INFO - Loading ./input.yaml for AutoRunner and making a copy in /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/work_dir/input.yaml\n", - "2022-09-28 16:23:16,977 - INFO - The output_dir is not specified. /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/work_dir/ensemble_output will be used to save ensemble predictions\n" + "2022-10-18 08:11:38,613 - INFO - Work directory ./work_dir is used to save all results\n", + "2022-10-18 08:11:38,615 - INFO - Loading ./input.yaml for AutoRunner and making a copy in /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/work_dir/input.yaml\n", + "2022-10-18 08:11:38,618 - INFO - The output_dir is not specified. /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/work_dir/ensemble_output will be used to save ensemble predictions\n" ] } ], @@ -423,9 +432,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "2022-09-28 16:23:17,289 - INFO - Work directory ./work_dir is used to save all results\n", - "2022-09-28 16:23:17,298 - INFO - Loading ./input.yaml for AutoRunner and making a copy in /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/work_dir/input.yaml\n", - "2022-09-28 16:23:17,329 - INFO - The output_dir is not specified. /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/work_dir/ensemble_output will be used to save ensemble predictions\n" + "2022-10-18 08:11:38,783 - INFO - Work directory ./work_dir is used to save all results\n", + "2022-10-18 08:11:38,784 - INFO - Loading ./input.yaml for AutoRunner and making a copy in /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/work_dir/input.yaml\n", + "2022-10-18 08:11:38,786 - INFO - The output_dir is not specified. /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/work_dir/ensemble_output will be used to save ensemble predictions\n" ] } ], @@ -462,9 +471,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "2022-09-28 16:23:17,663 - INFO - Work directory ./work_dir is used to save all results\n", - "2022-09-28 16:23:17,746 - INFO - Loading ./input.yaml for AutoRunner and making a copy in /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/work_dir/input.yaml\n", - "2022-09-28 16:23:17,765 - INFO - The output_dir is not specified. /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/work_dir/ensemble_output will be used to save ensemble predictions\n" + "2022-10-18 08:11:38,907 - INFO - Work directory ./work_dir is used to save all results\n", + "2022-10-18 08:11:38,908 - INFO - Loading ./input.yaml for AutoRunner and making a copy in /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/work_dir/input.yaml\n", + "2022-10-18 08:11:38,910 - INFO - The output_dir is not specified. /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/work_dir/ensemble_output will be used to save ensemble predictions\n" ] } ], @@ -506,9 +515,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "2022-09-28 16:23:18,053 - INFO - Work directory ./work_dir is used to save all results\n", - "2022-09-28 16:23:18,063 - INFO - Loading ./input.yaml for AutoRunner and making a copy in /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/work_dir/input.yaml\n", - "2022-09-28 16:23:18,084 - INFO - The output_dir is not specified. /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/work_dir/ensemble_output will be used to save ensemble predictions\n" + "2022-10-18 08:11:39,042 - INFO - Work directory ./work_dir is used to save all results\n", + "2022-10-18 08:11:39,046 - INFO - Loading ./input.yaml for AutoRunner and making a copy in /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/work_dir/input.yaml\n", + "2022-10-18 08:11:39,054 - INFO - The output_dir is not specified. /workspace/monai/tutorials-in-dev/auto3dseg/notebooks/work_dir/ensemble_output will be used to save ensemble predictions\n" ] } ], diff --git a/auto3dseg/notebooks/ensemble_byoc.ipynb b/auto3dseg/notebooks/ensemble_byoc.ipynb index 4d19f7f621..0e2926f634 100644 --- a/auto3dseg/notebooks/ensemble_byoc.ipynb +++ b/auto3dseg/notebooks/ensemble_byoc.ipynb @@ -14,7 +14,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -30,18 +30,9 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 7, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/opt/conda/lib/python3.8/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n" - ] - } - ], + "outputs": [], "source": [ "import os\n", "import numpy as np\n", @@ -49,6 +40,8 @@ "import random\n", "\n", "from copy import deepcopy\n", + "from pathlib import Path\n", + "\n", "from monai.apps.auto3dseg import (\n", " AlgoEnsemble,\n", " AlgoEnsembleBuilder,\n", @@ -70,7 +63,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -92,8 +85,8 @@ " ],\n", "}\n", "\n", - "dataroot = os.path.join(\"./data\")\n", - "work_dir = os.path.join(\"./ensemble_byoc_work_dir\")\n", + "dataroot = str(Path(\"./data\"))\n", + "work_dir = str(Path(\"./ensemble_byoc_work_dir\"))\n", "\n", "da_output_yaml = os.path.join(work_dir, \"datastats.yaml\")\n", "data_src_cfg = os.path.join(work_dir, \"data_src_cfg.yaml\")\n", @@ -130,7 +123,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -168,24 +161,25 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 10, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "100%|██████████| 12/12 [00:01<00:00, 9.60it/s]\n", - "algo_templates.tar.gz: 100%|██████████| 280k/280k [00:01<00:00, 172kB/s] " + "File ensemble_byoc_work_dir/datastats.yaml already exists and will be overwritten.\n", + "100%|██████████| 12/12 [00:00<00:00, 32.94it/s]\n", + "algo_templates.tar.gz: 296kB [00:01, 231kB/s] " ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2022-09-15 18:01:04,601 - INFO - Downloaded: /tmp/tmp55bhww8u/algo_templates.tar.gz\n", - "2022-09-15 18:01:04,602 - INFO - Expected md5 is None, skip md5 check for file /tmp/tmp55bhww8u/algo_templates.tar.gz.\n", - "2022-09-15 18:01:04,603 - INFO - Writing into directory: ./workdir.\n" + "2022-10-18 08:32:24,713 - INFO - Downloaded: /tmp/tmp9sgpyem1/algo_templates.tar.gz\n", + "2022-10-18 08:32:24,715 - INFO - Expected md5 is None, skip md5 check for file /tmp/tmp9sgpyem1/algo_templates.tar.gz.\n", + "2022-10-18 08:32:24,717 - INFO - Writing into directory: ensemble_byoc_work_dir.\n" ] }, { @@ -199,16 +193,16 @@ "name": "stdout", "output_type": "stream", "text": [ - "2022-09-15 18:01:04,892 - INFO - ./workdir/segresnet2d_0\n", - "2022-09-15 18:01:05,222 - INFO - ./workdir/segresnet2d_1\n", - "2022-09-15 18:01:05,450 - INFO - ./workdir/dints_0\n", - "2022-09-15 18:01:05,671 - INFO - ./workdir/dints_1\n", - "2022-09-15 18:01:05,897 - INFO - ./workdir/swinunetr_0\n", - "2022-09-15 18:01:06,145 - INFO - ./workdir/swinunetr_1\n", - "2022-09-15 18:01:06,376 - INFO - ./workdir/segresnet_0\n", - "2022-09-15 18:01:06,609 - INFO - ./workdir/segresnet_1\n", - "2022-09-15 18:01:06,610 - INFO - Launching: python ./workdir/segresnet2d_0/scripts/train.py run --config_file='./workdir/segresnet2d_0/configs/transforms_train.yaml','./workdir/segresnet2d_0/configs/network.yaml','./workdir/segresnet2d_0/configs/transforms_validate.yaml','./workdir/segresnet2d_0/configs/hyper_parameters.yaml','./workdir/segresnet2d_0/configs/transforms_infer.yaml' --num_iterations=8 --num_iterations_per_validation=4 --num_images_per_batch=2 --num_epochs=2 --num_warmup_iterations=4\n", - "2022-09-15 18:01:20,696 - INFO - CompletedProcess(args=['python', './workdir/segresnet2d_0/scripts/train.py', 'run', \"--config_file='./workdir/segresnet2d_0/configs/transforms_train.yaml','./workdir/segresnet2d_0/configs/network.yaml','./workdir/segresnet2d_0/configs/transforms_validate.yaml','./workdir/segresnet2d_0/configs/hyper_parameters.yaml','./workdir/segresnet2d_0/configs/transforms_infer.yaml'\", '--num_iterations=8', '--num_iterations_per_validation=4', '--num_images_per_batch=2', '--num_epochs=2', '--num_warmup_iterations=4'], returncode=0, stdout=b\"[info] number of GPUs: 1\n", + "2022-10-18 08:32:24,920 - INFO - ensemble_byoc_work_dir/segresnet2d_0\n", + "2022-10-18 08:32:25,037 - INFO - ensemble_byoc_work_dir/segresnet2d_1\n", + "2022-10-18 08:32:25,267 - INFO - ensemble_byoc_work_dir/dints_0\n", + "2022-10-18 08:32:25,391 - INFO - ensemble_byoc_work_dir/dints_1\n", + "2022-10-18 08:32:25,505 - INFO - ensemble_byoc_work_dir/swinunetr_0\n", + "2022-10-18 08:32:25,623 - INFO - ensemble_byoc_work_dir/swinunetr_1\n", + "2022-10-18 08:32:25,782 - INFO - ensemble_byoc_work_dir/segresnet_0\n", + "2022-10-18 08:32:25,905 - INFO - ensemble_byoc_work_dir/segresnet_1\n", + "2022-10-18 08:32:25,905 - INFO - Launching: python ensemble_byoc_work_dir/segresnet2d_0/scripts/train.py run --config_file='ensemble_byoc_work_dir/segresnet2d_0/configs/transforms_validate.yaml','ensemble_byoc_work_dir/segresnet2d_0/configs/transforms_train.yaml','ensemble_byoc_work_dir/segresnet2d_0/configs/transforms_infer.yaml','ensemble_byoc_work_dir/segresnet2d_0/configs/network.yaml','ensemble_byoc_work_dir/segresnet2d_0/configs/hyper_parameters.yaml' --num_iterations=8 --num_iterations_per_validation=4 --num_images_per_batch=2 --num_epochs=2 --num_warmup_iterations=4\n", + "2022-10-18 08:32:33,171 - INFO - CompletedProcess(args=['python', 'ensemble_byoc_work_dir/segresnet2d_0/scripts/train.py', 'run', \"--config_file='ensemble_byoc_work_dir/segresnet2d_0/configs/transforms_validate.yaml','ensemble_byoc_work_dir/segresnet2d_0/configs/transforms_train.yaml','ensemble_byoc_work_dir/segresnet2d_0/configs/transforms_infer.yaml','ensemble_byoc_work_dir/segresnet2d_0/configs/network.yaml','ensemble_byoc_work_dir/segresnet2d_0/configs/hyper_parameters.yaml'\", '--num_iterations=8', '--num_iterations_per_validation=4', '--num_images_per_batch=2', '--num_epochs=2', '--num_warmup_iterations=4'], returncode=0, stdout=b\"[info] number of GPUs: 1\n", "[info] world_size: 1\n", "train_files: 8\n", "val_files: 4\n", @@ -218,41 +212,40 @@ "[info] amp enabled\n", "----------\n", "epoch 1/2\n", - "learning rate is set to 0.025\n", - "[2022-09-15 18:01:14] 1/4, train_loss: 0.6757\n", - "[2022-09-15 18:01:14] 2/4, train_loss: 0.7041\n", - "[2022-09-15 18:01:14] 3/4, train_loss: 0.6267\n", - "[2022-09-15 18:01:14] 4/4, train_loss: 0.6508\n", - "epoch 1 average loss: 0.6643, best mean dice: -1.0000 at epoch -1\n", - "1 / 4 tensor([[0.2733]], device='cuda:0')\n", - "2 / 4 tensor([[0.1997]], device='cuda:0')\n", - "3 / 4 tensor([[0.2798]], device='cuda:0')\n", - "4 / 4 tensor([[0.2062]], device='cuda:0')\n", - "evaluation metric - class 1: 0.23973727226257324\n", - "avg_metric 0.23973727226257324\n", + "learning rate is set to 0.2\n", + "[2022-10-18 08:32:30] 1/4, train_loss: 0.5927\n", + "[2022-10-18 08:32:30] 2/4, train_loss: 0.6445\n", + "[2022-10-18 08:32:30] 3/4, train_loss: 0.5228\n", + "[2022-10-18 08:32:30] 4/4, train_loss: 0.3759\n", + "epoch 1 average loss: 0.5340, best mean dice: -1.0000 at epoch -1\n", + "1 / 4 tensor([[0.6033]], device='cuda:0')\n", + "2 / 4 tensor([[0.6033]], device='cuda:0')\n", + "3 / 4 tensor([[0.6033]], device='cuda:0')\n", + "4 / 4 tensor([[0.6033]], device='cuda:0')\n", + "evaluation metric - class 1: 0.603296160697937\n", + "avg_metric 0.603296160697937\n", "saved new best metric model\n", - "current epoch: 1 current mean dice: 0.2397 best mean dice: 0.2397 at epoch 1\n", + "current epoch: 1 current mean dice: 0.6033 best mean dice: 0.6033 at epoch 1\n", "----------\n", "epoch 2/2\n", - "learning rate is set to 0.0015625\n", - "[2022-09-15 18:01:17] 1/4, train_loss: 0.6124\n", - "[2022-09-15 18:01:17] 2/4, train_loss: 0.7155\n", - "[2022-09-15 18:01:17] 3/4, train_loss: 0.5988\n", - "[2022-09-15 18:01:17] 4/4, train_loss: 0.6238\n", - "epoch 2 average loss: 0.6376, best mean dice: 0.2397 at epoch 1\n", - "1 / 4 tensor([[0.3020]], device='cuda:0')\n", - "2 / 4 tensor([[0.2335]], device='cuda:0')\n", - "3 / 4 tensor([[0.3266]], device='cuda:0')\n", - "4 / 4 tensor([[0.2320]], device='cuda:0')\n", - "evaluation metric - class 1: 0.27353107929229736\n", - "avg_metric 0.27353107929229736\n", - "saved new best metric model\n", - "current epoch: 2 current mean dice: 0.2735 best mean dice: 0.2735 at epoch 2\n", - "train completed, best_metric: 0.2735 at epoch: 2\n", - "0.27353107929229736\n", + "learning rate is set to 0.0125\n", + "[2022-10-18 08:32:31] 1/4, train_loss: 0.4072\n", + "[2022-10-18 08:32:31] 2/4, train_loss: 0.3678\n", + "[2022-10-18 08:32:31] 3/4, train_loss: 0.3456\n", + "[2022-10-18 08:32:31] 4/4, train_loss: 0.3824\n", + "epoch 2 average loss: 0.3757, best mean dice: 0.6033 at epoch 1\n", + "1 / 4 tensor([[0.6025]], device='cuda:0')\n", + "2 / 4 tensor([[0.6025]], device='cuda:0')\n", + "3 / 4 tensor([[0.6025]], device='cuda:0')\n", + "4 / 4 tensor([[0.6025]], device='cuda:0')\n", + "evaluation metric - class 1: 0.6024555563926697\n", + "avg_metric 0.6024555563926697\n", + "current epoch: 2 current mean dice: 0.6025 best mean dice: 0.6033 at epoch 1\n", + "train completed, best_metric: 0.6033 at epoch: 1\n", + "0.603296160697937\n", "\", stderr=b'')\n", - "2022-09-15 18:01:20,697 - INFO - Launching: python ./workdir/segresnet2d_1/scripts/train.py run --config_file='./workdir/segresnet2d_1/configs/transforms_train.yaml','./workdir/segresnet2d_1/configs/network.yaml','./workdir/segresnet2d_1/configs/transforms_validate.yaml','./workdir/segresnet2d_1/configs/hyper_parameters.yaml','./workdir/segresnet2d_1/configs/transforms_infer.yaml' --num_iterations=8 --num_iterations_per_validation=4 --num_images_per_batch=2 --num_epochs=2 --num_warmup_iterations=4\n", - "2022-09-15 18:01:34,632 - INFO - CompletedProcess(args=['python', './workdir/segresnet2d_1/scripts/train.py', 'run', \"--config_file='./workdir/segresnet2d_1/configs/transforms_train.yaml','./workdir/segresnet2d_1/configs/network.yaml','./workdir/segresnet2d_1/configs/transforms_validate.yaml','./workdir/segresnet2d_1/configs/hyper_parameters.yaml','./workdir/segresnet2d_1/configs/transforms_infer.yaml'\", '--num_iterations=8', '--num_iterations_per_validation=4', '--num_images_per_batch=2', '--num_epochs=2', '--num_warmup_iterations=4'], returncode=0, stdout=b\"[info] number of GPUs: 1\n", + "2022-10-18 08:32:33,172 - INFO - Launching: python ensemble_byoc_work_dir/segresnet2d_1/scripts/train.py run --config_file='ensemble_byoc_work_dir/segresnet2d_1/configs/transforms_validate.yaml','ensemble_byoc_work_dir/segresnet2d_1/configs/transforms_train.yaml','ensemble_byoc_work_dir/segresnet2d_1/configs/transforms_infer.yaml','ensemble_byoc_work_dir/segresnet2d_1/configs/network.yaml','ensemble_byoc_work_dir/segresnet2d_1/configs/hyper_parameters.yaml' --num_iterations=8 --num_iterations_per_validation=4 --num_images_per_batch=2 --num_epochs=2 --num_warmup_iterations=4\n", + "2022-10-18 08:32:40,511 - INFO - CompletedProcess(args=['python', 'ensemble_byoc_work_dir/segresnet2d_1/scripts/train.py', 'run', \"--config_file='ensemble_byoc_work_dir/segresnet2d_1/configs/transforms_validate.yaml','ensemble_byoc_work_dir/segresnet2d_1/configs/transforms_train.yaml','ensemble_byoc_work_dir/segresnet2d_1/configs/transforms_infer.yaml','ensemble_byoc_work_dir/segresnet2d_1/configs/network.yaml','ensemble_byoc_work_dir/segresnet2d_1/configs/hyper_parameters.yaml'\", '--num_iterations=8', '--num_iterations_per_validation=4', '--num_images_per_batch=2', '--num_epochs=2', '--num_warmup_iterations=4'], returncode=0, stdout=b\"[info] number of GPUs: 1\n", "[info] world_size: 1\n", "train_files: 8\n", "val_files: 4\n", @@ -262,41 +255,41 @@ "[info] amp enabled\n", "----------\n", "epoch 1/2\n", - "learning rate is set to 0.025\n", - "[2022-09-15 18:01:27] 1/4, train_loss: 0.5915\n", - "[2022-09-15 18:01:27] 2/4, train_loss: 0.5380\n", - "[2022-09-15 18:01:27] 3/4, train_loss: 0.4677\n", - "[2022-09-15 18:01:27] 4/4, train_loss: 0.4263\n", - "epoch 1 average loss: 0.5059, best mean dice: -1.0000 at epoch -1\n", - "1 / 4 tensor([[0.5196]], device='cuda:0')\n", - "2 / 4 tensor([[0.5353]], device='cuda:0')\n", - "3 / 4 tensor([[0.5120]], device='cuda:0')\n", - "4 / 4 tensor([[0.5498]], device='cuda:0')\n", - "evaluation metric - class 1: 0.5291600227355957\n", - "avg_metric 0.5291600227355957\n", + "learning rate is set to 0.2\n", + "[2022-10-18 08:32:37] 1/4, train_loss: 0.6737\n", + "[2022-10-18 08:32:37] 2/4, train_loss: 0.5555\n", + "[2022-10-18 08:32:37] 3/4, train_loss: 0.5402\n", + "[2022-10-18 08:32:37] 4/4, train_loss: 0.4564\n", + "epoch 1 average loss: 0.5565, best mean dice: -1.0000 at epoch -1\n", + "1 / 4 tensor([[0.5682]], device='cuda:0')\n", + "2 / 4 tensor([[0.5682]], device='cuda:0')\n", + "3 / 4 tensor([[0.5682]], device='cuda:0')\n", + "4 / 4 tensor([[0.5682]], device='cuda:0')\n", + "evaluation metric - class 1: 0.5681775808334351\n", + "avg_metric 0.5681775808334351\n", "saved new best metric model\n", - "current epoch: 1 current mean dice: 0.5292 best mean dice: 0.5292 at epoch 1\n", + "current epoch: 1 current mean dice: 0.5682 best mean dice: 0.5682 at epoch 1\n", "----------\n", "epoch 2/2\n", - "learning rate is set to 0.0015625\n", - "[2022-09-15 18:01:31] 1/4, train_loss: 0.4483\n", - "[2022-09-15 18:01:31] 2/4, train_loss: 0.4215\n", - "[2022-09-15 18:01:31] 3/4, train_loss: 0.4549\n", - "[2022-09-15 18:01:31] 4/4, train_loss: 0.5297\n", - "epoch 2 average loss: 0.4636, best mean dice: 0.5292 at epoch 1\n", - "1 / 4 tensor([[0.5413]], device='cuda:0')\n", - "2 / 4 tensor([[0.5535]], device='cuda:0')\n", - "3 / 4 tensor([[0.5337]], device='cuda:0')\n", - "4 / 4 tensor([[0.5665]], device='cuda:0')\n", - "evaluation metric - class 1: 0.5487756729125977\n", - "avg_metric 0.5487756729125977\n", + "learning rate is set to 0.0125\n", + "[2022-10-18 08:32:38] 1/4, train_loss: 0.4514\n", + "[2022-10-18 08:32:38] 2/4, train_loss: 0.6421\n", + "[2022-10-18 08:32:39] 3/4, train_loss: 0.4087\n", + "[2022-10-18 08:32:39] 4/4, train_loss: 0.4259\n", + "epoch 2 average loss: 0.4820, best mean dice: 0.5682 at epoch 1\n", + "1 / 4 tensor([[0.5707]], device='cuda:0')\n", + "2 / 4 tensor([[0.5707]], device='cuda:0')\n", + "3 / 4 tensor([[0.5707]], device='cuda:0')\n", + "4 / 4 tensor([[0.5707]], device='cuda:0')\n", + "evaluation metric - class 1: 0.5707199573516846\n", + "avg_metric 0.5707199573516846\n", "saved new best metric model\n", - "current epoch: 2 current mean dice: 0.5488 best mean dice: 0.5488 at epoch 2\n", - "train completed, best_metric: 0.5488 at epoch: 2\n", - "0.5487756729125977\n", + "current epoch: 2 current mean dice: 0.5707 best mean dice: 0.5707 at epoch 2\n", + "train completed, best_metric: 0.5707 at epoch: 2\n", + "0.5707199573516846\n", "\", stderr=b'')\n", - "2022-09-15 18:01:34,634 - INFO - Launching: python ./workdir/dints_0/scripts/search.py run --config_file='./workdir/dints_0/configs/transforms_train.yaml','./workdir/dints_0/configs/network.yaml','./workdir/dints_0/configs/transforms_validate.yaml','./workdir/dints_0/configs/hyper_parameters_search.yaml','./workdir/dints_0/configs/network_search.yaml','./workdir/dints_0/configs/hyper_parameters.yaml','./workdir/dints_0/configs/transforms_infer.yaml' --searching#num_iterations=8 --searching#num_iterations_per_validation=4 --searching#num_images_per_batch=2 --searching#num_epochs=2 --searching#num_warmup_iterations=4\n", - "2022-09-15 18:02:23,660 - INFO - CompletedProcess(args=['python', './workdir/dints_0/scripts/search.py', 'run', \"--config_file='./workdir/dints_0/configs/transforms_train.yaml','./workdir/dints_0/configs/network.yaml','./workdir/dints_0/configs/transforms_validate.yaml','./workdir/dints_0/configs/hyper_parameters_search.yaml','./workdir/dints_0/configs/network_search.yaml','./workdir/dints_0/configs/hyper_parameters.yaml','./workdir/dints_0/configs/transforms_infer.yaml'\", '--searching#num_iterations=8', '--searching#num_iterations_per_validation=4', '--searching#num_images_per_batch=2', '--searching#num_epochs=2', '--searching#num_warmup_iterations=4'], returncode=0, stdout=b\"[info] number of GPUs: 1\n", + "2022-10-18 08:32:40,512 - INFO - Launching: python ensemble_byoc_work_dir/dints_0/scripts/search.py run --config_file='ensemble_byoc_work_dir/dints_0/configs/transforms_validate.yaml','ensemble_byoc_work_dir/dints_0/configs/transforms_train.yaml','ensemble_byoc_work_dir/dints_0/configs/transforms_infer.yaml','ensemble_byoc_work_dir/dints_0/configs/hyper_parameters_search.yaml','ensemble_byoc_work_dir/dints_0/configs/network.yaml','ensemble_byoc_work_dir/dints_0/configs/network_search.yaml','ensemble_byoc_work_dir/dints_0/configs/hyper_parameters.yaml' --searching#num_iterations=8 --searching#num_iterations_per_validation=4 --searching#num_images_per_batch=2 --searching#num_epochs=2 --searching#num_warmup_iterations=4\n", + "2022-10-18 08:33:08,731 - INFO - CompletedProcess(args=['python', 'ensemble_byoc_work_dir/dints_0/scripts/search.py', 'run', \"--config_file='ensemble_byoc_work_dir/dints_0/configs/transforms_validate.yaml','ensemble_byoc_work_dir/dints_0/configs/transforms_train.yaml','ensemble_byoc_work_dir/dints_0/configs/transforms_infer.yaml','ensemble_byoc_work_dir/dints_0/configs/hyper_parameters_search.yaml','ensemble_byoc_work_dir/dints_0/configs/network.yaml','ensemble_byoc_work_dir/dints_0/configs/network_search.yaml','ensemble_byoc_work_dir/dints_0/configs/hyper_parameters.yaml'\", '--searching#num_iterations=8', '--searching#num_iterations_per_validation=4', '--searching#num_images_per_batch=2', '--searching#num_epochs=2', '--searching#num_warmup_iterations=4'], returncode=0, stdout=b\"[info] number of GPUs: 1\n", "[info] world_size: 1\n", "train_files_w: 4\n", "train_files_a: 4\n", @@ -308,53 +301,53 @@ "----------\n", "epoch 1/4\n", "learning rate is set to 0.025\n", - "[2022-09-15 18:01:44] 1/2, train_loss: 0.6853\n", - "[2022-09-15 18:01:45] 2/2, train_loss: 0.6995\n", - "epoch 1 average loss: 0.6924, best mean dice: -1.0000 at epoch -1\n", + "[2022-10-18 08:32:46] 1/2, train_loss: 0.7129\n", + "[2022-10-18 08:32:46] 2/2, train_loss: 0.7068\n", + "epoch 1 average loss: 0.7099, best mean dice: -1.0000 at epoch -1\n", "----------\n", "epoch 2/4\n", "learning rate is set to 0.025\n", - "[2022-09-15 18:01:47] 1/2, train_loss: 0.6358\n", - "[2022-09-15 18:01:48] 2/2, train_loss: 0.7008\n", - "epoch 2 average loss: 0.6683, best mean dice: -1.0000 at epoch -1\n", - "1 / 4 tensor([[0.1477]], device='cuda:0')\n", - "2 / 4 tensor([[0.1167]], device='cuda:0')\n", - "3 / 4 tensor([[0.1947]], device='cuda:0')\n", - "4 / 4 tensor([[0.1360]], device='cuda:0')\n", - "evaluation metric - class 1: 0.1487857848405838\n", - "avg_metric 0.1487857848405838\n", + "[2022-10-18 08:32:47] 1/2, train_loss: 0.6781\n", + "[2022-10-18 08:32:48] 2/2, train_loss: 0.6687\n", + "epoch 2 average loss: 0.6734, best mean dice: -1.0000 at epoch -1\n", + "1 / 4 tensor([[0.2709]], device='cuda:0')\n", + "2 / 4 tensor([[0.2709]], device='cuda:0')\n", + "3 / 4 tensor([[0.2709]], device='cuda:0')\n", + "4 / 4 tensor([[0.2709]], device='cuda:0')\n", + "evaluation metric - class 1: 0.2709421217441559\n", + "avg_metric 0.2709421217441559\n", "saved new best metric model\n", - "current epoch: 2 current mean dice: 0.1488 best mean dice: 0.1488 at epoch 2\n", + "current epoch: 2 current mean dice: 0.2709 best mean dice: 0.2709 at epoch 2\n", "----------\n", "epoch 3/4\n", "learning rate is set to 0.025\n", - "[2022-09-15 18:02:02] 1/2, train_loss: 0.6778\n", - "[2022-09-15 18:02:03] 1/2, train_loss_arch: 0.6921\n", - "[2022-09-15 18:02:04] 2/2, train_loss: 0.6846\n", - "[2022-09-15 18:02:06] 2/2, train_loss_arch: 0.6744\n", - "epoch 3 average loss: 0.6812, best mean dice: 0.1488 at epoch 2\n", - "epoch 3 average arch loss: 0.6833, best mean dice: 0.1488 at epoch 2\n", + "[2022-10-18 08:32:56] 1/2, train_loss: 0.6463\n", + "[2022-10-18 08:32:57] 1/2, train_loss_arch: 0.6450\n", + "[2022-10-18 08:32:57] 2/2, train_loss: 0.6100\n", + "[2022-10-18 08:32:58] 2/2, train_loss_arch: 0.6864\n", + "epoch 3 average loss: 0.6281, best mean dice: 0.2709 at epoch 2\n", + "epoch 3 average arch loss: 0.6657, best mean dice: 0.2709 at epoch 2\n", "----------\n", "epoch 4/4\n", "learning rate is set to 0.00625\n", - "[2022-09-15 18:02:07] 1/2, train_loss: 0.6759\n", - "[2022-09-15 18:02:09] 1/2, train_loss_arch: 0.9960\n", - "[2022-09-15 18:02:10] 2/2, train_loss: 0.6710\n", - "[2022-09-15 18:02:11] 2/2, train_loss_arch: 1.0121\n", - "epoch 4 average loss: 0.6734, best mean dice: 0.1488 at epoch 2\n", - "epoch 4 average arch loss: 1.0040, best mean dice: 0.1488 at epoch 2\n", - "1 / 4 tensor([[0.1581]], device='cuda:0')\n", - "2 / 4 tensor([[0.1656]], device='cuda:0')\n", - "3 / 4 tensor([[0.2173]], device='cuda:0')\n", - "4 / 4 tensor([[0.1385]], device='cuda:0')\n", - "evaluation metric - class 1: 0.1698775291442871\n", - "avg_metric 0.1698775291442871\n", + "[2022-10-18 08:32:59] 1/2, train_loss: 0.6752\n", + "[2022-10-18 08:33:00] 1/2, train_loss_arch: 0.9512\n", + "[2022-10-18 08:33:00] 2/2, train_loss: 0.6497\n", + "[2022-10-18 08:33:01] 2/2, train_loss_arch: 0.9469\n", + "epoch 4 average loss: 0.6624, best mean dice: 0.2709 at epoch 2\n", + "epoch 4 average arch loss: 0.9491, best mean dice: 0.2709 at epoch 2\n", + "1 / 4 tensor([[0.3194]], device='cuda:0')\n", + "2 / 4 tensor([[0.3194]], device='cuda:0')\n", + "3 / 4 tensor([[0.3194]], device='cuda:0')\n", + "4 / 4 tensor([[0.3194]], device='cuda:0')\n", + "evaluation metric - class 1: 0.31935712695121765\n", + "avg_metric 0.31935712695121765\n", "saved new best metric model\n", - "current epoch: 4 current mean dice: 0.1699 best mean dice: 0.1699 at epoch 4\n", - "train completed, best_metric: 0.1699 at epoch: 4\n", + "current epoch: 4 current mean dice: 0.3194 best mean dice: 0.3194 at epoch 4\n", + "train completed, best_metric: 0.3194 at epoch: 4\n", "\", stderr=b'')\n", - "2022-09-15 18:02:23,661 - INFO - Launching: python ./workdir/dints_0/scripts/train.py run --config_file='./workdir/dints_0/configs/transforms_train.yaml','./workdir/dints_0/configs/network.yaml','./workdir/dints_0/configs/transforms_validate.yaml','./workdir/dints_0/configs/hyper_parameters_search.yaml','./workdir/dints_0/configs/network_search.yaml','./workdir/dints_0/configs/hyper_parameters.yaml','./workdir/dints_0/configs/transforms_infer.yaml' --training#num_iterations=8 --training#num_iterations_per_validation=4 --training#num_images_per_batch=2 --training#num_epochs=2 --training#num_warmup_iterations=4\n", - "2022-09-15 18:02:41,747 - INFO - CompletedProcess(args=['python', './workdir/dints_0/scripts/train.py', 'run', \"--config_file='./workdir/dints_0/configs/transforms_train.yaml','./workdir/dints_0/configs/network.yaml','./workdir/dints_0/configs/transforms_validate.yaml','./workdir/dints_0/configs/hyper_parameters_search.yaml','./workdir/dints_0/configs/network_search.yaml','./workdir/dints_0/configs/hyper_parameters.yaml','./workdir/dints_0/configs/transforms_infer.yaml'\", '--training#num_iterations=8', '--training#num_iterations_per_validation=4', '--training#num_images_per_batch=2', '--training#num_epochs=2', '--training#num_warmup_iterations=4'], returncode=0, stdout=b\"[info] number of GPUs: 1\n", + "2022-10-18 08:33:08,732 - INFO - Launching: python ensemble_byoc_work_dir/dints_0/scripts/train.py run --config_file='ensemble_byoc_work_dir/dints_0/configs/transforms_validate.yaml','ensemble_byoc_work_dir/dints_0/configs/transforms_train.yaml','ensemble_byoc_work_dir/dints_0/configs/transforms_infer.yaml','ensemble_byoc_work_dir/dints_0/configs/hyper_parameters_search.yaml','ensemble_byoc_work_dir/dints_0/configs/network.yaml','ensemble_byoc_work_dir/dints_0/configs/network_search.yaml','ensemble_byoc_work_dir/dints_0/configs/hyper_parameters.yaml' --training#num_iterations=8 --training#num_iterations_per_validation=4 --training#num_images_per_batch=2 --training#num_epochs=2 --training#num_warmup_iterations=4\n", + "2022-10-18 08:33:19,665 - INFO - CompletedProcess(args=['python', 'ensemble_byoc_work_dir/dints_0/scripts/train.py', 'run', \"--config_file='ensemble_byoc_work_dir/dints_0/configs/transforms_validate.yaml','ensemble_byoc_work_dir/dints_0/configs/transforms_train.yaml','ensemble_byoc_work_dir/dints_0/configs/transforms_infer.yaml','ensemble_byoc_work_dir/dints_0/configs/hyper_parameters_search.yaml','ensemble_byoc_work_dir/dints_0/configs/network.yaml','ensemble_byoc_work_dir/dints_0/configs/network_search.yaml','ensemble_byoc_work_dir/dints_0/configs/hyper_parameters.yaml'\", '--training#num_iterations=8', '--training#num_iterations_per_validation=4', '--training#num_images_per_batch=2', '--training#num_epochs=2', '--training#num_warmup_iterations=4'], returncode=0, stdout=b\"[info] number of GPUs: 1\n", "[info] world_size: 1\n", "train_files: 8\n", "val_files: 4\n", @@ -365,40 +358,40 @@ "----------\n", "epoch 1/2\n", "learning rate is set to 0.2\n", - "[2022-09-15 18:02:33] 1/4, train_loss: 0.8255\n", - "[2022-09-15 18:02:33] 2/4, train_loss: 0.7661\n", - "[2022-09-15 18:02:34] 3/4, train_loss: 0.7437\n", - "[2022-09-15 18:02:34] 4/4, train_loss: 0.7193\n", - "epoch 1 average loss: 0.7636, best mean dice: -1.0000 at epoch -1\n", - "1 / 4 tensor([[0.1721]], device='cuda:0')\n", - "2 / 4 tensor([[0.1961]], device='cuda:0')\n", - "3 / 4 tensor([[0.2230]], device='cuda:0')\n", - "4 / 4 tensor([[0.1884]], device='cuda:0')\n", - "evaluation metric - class 1: 0.1948952078819275\n", - "avg_metric 0.1948952078819275\n", + "[2022-10-18 08:33:14] 1/4, train_loss: 0.7762\n", + "[2022-10-18 08:33:14] 2/4, train_loss: 0.7486\n", + "[2022-10-18 08:33:15] 3/4, train_loss: 0.7146\n", + "[2022-10-18 08:33:15] 4/4, train_loss: 0.7077\n", + "epoch 1 average loss: 0.7368, best mean dice: -1.0000 at epoch -1\n", + "1 / 4 tensor([[0.2790]], device='cuda:0')\n", + "2 / 4 tensor([[0.2790]], device='cuda:0')\n", + "3 / 4 tensor([[0.2790]], device='cuda:0')\n", + "4 / 4 tensor([[0.2790]], device='cuda:0')\n", + "evaluation metric - class 1: 0.27896663546562195\n", + "avg_metric 0.27896663546562195\n", "saved new best metric model\n", - "current epoch: 1 current mean dice: 0.1949 best mean dice: 0.1949 at epoch 1\n", + "current epoch: 1 current mean dice: 0.2790 best mean dice: 0.2790 at epoch 1\n", "----------\n", "epoch 2/2\n", "learning rate is set to 0.0125\n", - "[2022-09-15 18:02:37] 1/4, train_loss: 0.6790\n", - "[2022-09-15 18:02:38] 2/4, train_loss: 0.7413\n", - "[2022-09-15 18:02:38] 3/4, train_loss: 0.7272\n", - "[2022-09-15 18:02:39] 4/4, train_loss: 0.6882\n", - "epoch 2 average loss: 0.7089, best mean dice: 0.1949 at epoch 1\n", - "1 / 4 tensor([[0.1653]], device='cuda:0')\n", - "2 / 4 tensor([[0.1973]], device='cuda:0')\n", - "3 / 4 tensor([[0.2316]], device='cuda:0')\n", - "4 / 4 tensor([[0.1857]], device='cuda:0')\n", - "evaluation metric - class 1: 0.1949695199728012\n", - "avg_metric 0.1949695199728012\n", + "[2022-10-18 08:33:16] 1/4, train_loss: 0.6856\n", + "[2022-10-18 08:33:17] 2/4, train_loss: 0.6843\n", + "[2022-10-18 08:33:17] 3/4, train_loss: 0.6850\n", + "[2022-10-18 08:33:18] 4/4, train_loss: 0.6781\n", + "epoch 2 average loss: 0.6832, best mean dice: 0.2790 at epoch 1\n", + "1 / 4 tensor([[0.2965]], device='cuda:0')\n", + "2 / 4 tensor([[0.2965]], device='cuda:0')\n", + "3 / 4 tensor([[0.2965]], device='cuda:0')\n", + "4 / 4 tensor([[0.2965]], device='cuda:0')\n", + "evaluation metric - class 1: 0.29653027653694153\n", + "avg_metric 0.29653027653694153\n", "saved new best metric model\n", - "current epoch: 2 current mean dice: 0.1950 best mean dice: 0.1950 at epoch 2\n", - "train completed, best_metric: 0.1950 at epoch: 2\n", - "0.1949695199728012\n", + "current epoch: 2 current mean dice: 0.2965 best mean dice: 0.2965 at epoch 2\n", + "train completed, best_metric: 0.2965 at epoch: 2\n", + "0.29653027653694153\n", "\", stderr=b'')\n", - "2022-09-15 18:02:41,749 - INFO - Launching: python ./workdir/dints_1/scripts/search.py run --config_file='./workdir/dints_1/configs/transforms_train.yaml','./workdir/dints_1/configs/network.yaml','./workdir/dints_1/configs/transforms_validate.yaml','./workdir/dints_1/configs/hyper_parameters_search.yaml','./workdir/dints_1/configs/network_search.yaml','./workdir/dints_1/configs/hyper_parameters.yaml','./workdir/dints_1/configs/transforms_infer.yaml' --searching#num_iterations=8 --searching#num_iterations_per_validation=4 --searching#num_images_per_batch=2 --searching#num_epochs=2 --searching#num_warmup_iterations=4\n", - "2022-09-15 18:03:31,385 - INFO - CompletedProcess(args=['python', './workdir/dints_1/scripts/search.py', 'run', \"--config_file='./workdir/dints_1/configs/transforms_train.yaml','./workdir/dints_1/configs/network.yaml','./workdir/dints_1/configs/transforms_validate.yaml','./workdir/dints_1/configs/hyper_parameters_search.yaml','./workdir/dints_1/configs/network_search.yaml','./workdir/dints_1/configs/hyper_parameters.yaml','./workdir/dints_1/configs/transforms_infer.yaml'\", '--searching#num_iterations=8', '--searching#num_iterations_per_validation=4', '--searching#num_images_per_batch=2', '--searching#num_epochs=2', '--searching#num_warmup_iterations=4'], returncode=0, stdout=b\"[info] number of GPUs: 1\n", + "2022-10-18 08:33:19,666 - INFO - Launching: python ensemble_byoc_work_dir/dints_1/scripts/search.py run --config_file='ensemble_byoc_work_dir/dints_1/configs/transforms_validate.yaml','ensemble_byoc_work_dir/dints_1/configs/transforms_train.yaml','ensemble_byoc_work_dir/dints_1/configs/transforms_infer.yaml','ensemble_byoc_work_dir/dints_1/configs/hyper_parameters_search.yaml','ensemble_byoc_work_dir/dints_1/configs/network.yaml','ensemble_byoc_work_dir/dints_1/configs/network_search.yaml','ensemble_byoc_work_dir/dints_1/configs/hyper_parameters.yaml' --searching#num_iterations=8 --searching#num_iterations_per_validation=4 --searching#num_images_per_batch=2 --searching#num_epochs=2 --searching#num_warmup_iterations=4\n", + "2022-10-18 08:33:47,988 - INFO - CompletedProcess(args=['python', 'ensemble_byoc_work_dir/dints_1/scripts/search.py', 'run', \"--config_file='ensemble_byoc_work_dir/dints_1/configs/transforms_validate.yaml','ensemble_byoc_work_dir/dints_1/configs/transforms_train.yaml','ensemble_byoc_work_dir/dints_1/configs/transforms_infer.yaml','ensemble_byoc_work_dir/dints_1/configs/hyper_parameters_search.yaml','ensemble_byoc_work_dir/dints_1/configs/network.yaml','ensemble_byoc_work_dir/dints_1/configs/network_search.yaml','ensemble_byoc_work_dir/dints_1/configs/hyper_parameters.yaml'\", '--searching#num_iterations=8', '--searching#num_iterations_per_validation=4', '--searching#num_images_per_batch=2', '--searching#num_epochs=2', '--searching#num_warmup_iterations=4'], returncode=0, stdout=b\"[info] number of GPUs: 1\n", "[info] world_size: 1\n", "train_files_w: 4\n", "train_files_a: 4\n", @@ -410,53 +403,53 @@ "----------\n", "epoch 1/4\n", "learning rate is set to 0.025\n", - "[2022-09-15 18:02:51] 1/2, train_loss: 0.6734\n", - "[2022-09-15 18:02:52] 2/2, train_loss: 0.6714\n", - "epoch 1 average loss: 0.6724, best mean dice: -1.0000 at epoch -1\n", + "[2022-10-18 08:33:25] 1/2, train_loss: 0.7205\n", + "[2022-10-18 08:33:26] 2/2, train_loss: 0.7164\n", + "epoch 1 average loss: 0.7185, best mean dice: -1.0000 at epoch -1\n", "----------\n", "epoch 2/4\n", "learning rate is set to 0.025\n", - "[2022-09-15 18:02:54] 1/2, train_loss: 0.6868\n", - "[2022-09-15 18:02:55] 2/2, train_loss: 0.6793\n", - "epoch 2 average loss: 0.6830, best mean dice: -1.0000 at epoch -1\n", - "1 / 4 tensor([[0.1239]], device='cuda:0')\n", - "2 / 4 tensor([[0.0854]], device='cuda:0')\n", - "3 / 4 tensor([[0.1178]], device='cuda:0')\n", - "4 / 4 tensor([[0.1838]], device='cuda:0')\n", - "evaluation metric - class 1: 0.1277436465024948\n", - "avg_metric 0.1277436465024948\n", + "[2022-10-18 08:33:27] 1/2, train_loss: 0.6990\n", + "[2022-10-18 08:33:27] 2/2, train_loss: 0.6014\n", + "epoch 2 average loss: 0.6502, best mean dice: -1.0000 at epoch -1\n", + "1 / 4 tensor([[0.2154]], device='cuda:0')\n", + "2 / 4 tensor([[0.2154]], device='cuda:0')\n", + "3 / 4 tensor([[0.2154]], device='cuda:0')\n", + "4 / 4 tensor([[0.2154]], device='cuda:0')\n", + "evaluation metric - class 1: 0.2154349982738495\n", + "avg_metric 0.2154349982738495\n", "saved new best metric model\n", - "current epoch: 2 current mean dice: 0.1277 best mean dice: 0.1277 at epoch 2\n", + "current epoch: 2 current mean dice: 0.2154 best mean dice: 0.2154 at epoch 2\n", "----------\n", "epoch 3/4\n", "learning rate is set to 0.025\n", - "[2022-09-15 18:03:08] 1/2, train_loss: 0.6641\n", - "[2022-09-15 18:03:10] 1/2, train_loss_arch: 0.6710\n", - "[2022-09-15 18:03:11] 2/2, train_loss: 0.6682\n", - "[2022-09-15 18:03:12] 2/2, train_loss_arch: 0.6540\n", - "epoch 3 average loss: 0.6661, best mean dice: 0.1277 at epoch 2\n", - "epoch 3 average arch loss: 0.6625, best mean dice: 0.1277 at epoch 2\n", + "[2022-10-18 08:33:35] 1/2, train_loss: 0.6494\n", + "[2022-10-18 08:33:36] 1/2, train_loss_arch: 0.6350\n", + "[2022-10-18 08:33:37] 2/2, train_loss: 0.6811\n", + "[2022-10-18 08:33:37] 2/2, train_loss_arch: 0.6500\n", + "epoch 3 average loss: 0.6653, best mean dice: 0.2154 at epoch 2\n", + "epoch 3 average arch loss: 0.6425, best mean dice: 0.2154 at epoch 2\n", "----------\n", "epoch 4/4\n", "learning rate is set to 0.00625\n", - "[2022-09-15 18:03:14] 1/2, train_loss: 0.6937\n", - "[2022-09-15 18:03:16] 1/2, train_loss_arch: 0.9790\n", - "[2022-09-15 18:03:17] 2/2, train_loss: 0.6240\n", - "[2022-09-15 18:03:18] 2/2, train_loss_arch: 0.9612\n", - "epoch 4 average loss: 0.6589, best mean dice: 0.1277 at epoch 2\n", - "epoch 4 average arch loss: 0.9701, best mean dice: 0.1277 at epoch 2\n", - "1 / 4 tensor([[0.1406]], device='cuda:0')\n", - "2 / 4 tensor([[0.1062]], device='cuda:0')\n", - "3 / 4 tensor([[0.1512]], device='cuda:0')\n", - "4 / 4 tensor([[0.1979]], device='cuda:0')\n", - "evaluation metric - class 1: 0.14896240830421448\n", - "avg_metric 0.14896240830421448\n", + "[2022-10-18 08:33:38] 1/2, train_loss: 0.6306\n", + "[2022-10-18 08:33:39] 1/2, train_loss_arch: 0.9758\n", + "[2022-10-18 08:33:40] 2/2, train_loss: 0.6403\n", + "[2022-10-18 08:33:40] 2/2, train_loss_arch: 0.9940\n", + "epoch 4 average loss: 0.6355, best mean dice: 0.2154 at epoch 2\n", + "epoch 4 average arch loss: 0.9849, best mean dice: 0.2154 at epoch 2\n", + "1 / 4 tensor([[0.2152]], device='cuda:0')\n", + "2 / 4 tensor([[0.2152]], device='cuda:0')\n", + "3 / 4 tensor([[0.2152]], device='cuda:0')\n", + "4 / 4 tensor([[0.2152]], device='cuda:0')\n", + "evaluation metric - class 1: 0.21520645916461945\n", + "avg_metric 0.21520645916461945\n", "saved new best metric model\n", - "current epoch: 4 current mean dice: 0.1490 best mean dice: 0.1490 at epoch 4\n", - "train completed, best_metric: 0.1490 at epoch: 4\n", + "current epoch: 4 current mean dice: 0.2152 best mean dice: 0.2154 at epoch 2\n", + "train completed, best_metric: 0.2154 at epoch: 2\n", "\", stderr=b'')\n", - "2022-09-15 18:03:31,386 - INFO - Launching: python ./workdir/dints_1/scripts/train.py run --config_file='./workdir/dints_1/configs/transforms_train.yaml','./workdir/dints_1/configs/network.yaml','./workdir/dints_1/configs/transforms_validate.yaml','./workdir/dints_1/configs/hyper_parameters_search.yaml','./workdir/dints_1/configs/network_search.yaml','./workdir/dints_1/configs/hyper_parameters.yaml','./workdir/dints_1/configs/transforms_infer.yaml' --training#num_iterations=8 --training#num_iterations_per_validation=4 --training#num_images_per_batch=2 --training#num_epochs=2 --training#num_warmup_iterations=4\n", - "2022-09-15 18:03:49,837 - INFO - CompletedProcess(args=['python', './workdir/dints_1/scripts/train.py', 'run', \"--config_file='./workdir/dints_1/configs/transforms_train.yaml','./workdir/dints_1/configs/network.yaml','./workdir/dints_1/configs/transforms_validate.yaml','./workdir/dints_1/configs/hyper_parameters_search.yaml','./workdir/dints_1/configs/network_search.yaml','./workdir/dints_1/configs/hyper_parameters.yaml','./workdir/dints_1/configs/transforms_infer.yaml'\", '--training#num_iterations=8', '--training#num_iterations_per_validation=4', '--training#num_images_per_batch=2', '--training#num_epochs=2', '--training#num_warmup_iterations=4'], returncode=0, stdout=b\"[info] number of GPUs: 1\n", + "2022-10-18 08:33:47,989 - INFO - Launching: python ensemble_byoc_work_dir/dints_1/scripts/train.py run --config_file='ensemble_byoc_work_dir/dints_1/configs/transforms_validate.yaml','ensemble_byoc_work_dir/dints_1/configs/transforms_train.yaml','ensemble_byoc_work_dir/dints_1/configs/transforms_infer.yaml','ensemble_byoc_work_dir/dints_1/configs/hyper_parameters_search.yaml','ensemble_byoc_work_dir/dints_1/configs/network.yaml','ensemble_byoc_work_dir/dints_1/configs/network_search.yaml','ensemble_byoc_work_dir/dints_1/configs/hyper_parameters.yaml' --training#num_iterations=8 --training#num_iterations_per_validation=4 --training#num_images_per_batch=2 --training#num_epochs=2 --training#num_warmup_iterations=4\n", + "2022-10-18 08:33:59,000 - INFO - CompletedProcess(args=['python', 'ensemble_byoc_work_dir/dints_1/scripts/train.py', 'run', \"--config_file='ensemble_byoc_work_dir/dints_1/configs/transforms_validate.yaml','ensemble_byoc_work_dir/dints_1/configs/transforms_train.yaml','ensemble_byoc_work_dir/dints_1/configs/transforms_infer.yaml','ensemble_byoc_work_dir/dints_1/configs/hyper_parameters_search.yaml','ensemble_byoc_work_dir/dints_1/configs/network.yaml','ensemble_byoc_work_dir/dints_1/configs/network_search.yaml','ensemble_byoc_work_dir/dints_1/configs/hyper_parameters.yaml'\", '--training#num_iterations=8', '--training#num_iterations_per_validation=4', '--training#num_images_per_batch=2', '--training#num_epochs=2', '--training#num_warmup_iterations=4'], returncode=0, stdout=b\"[info] number of GPUs: 1\n", "[info] world_size: 1\n", "train_files: 8\n", "val_files: 4\n", @@ -467,40 +460,40 @@ "----------\n", "epoch 1/2\n", "learning rate is set to 0.2\n", - "[2022-09-15 18:03:40] 1/4, train_loss: 0.7471\n", - "[2022-09-15 18:03:41] 2/4, train_loss: 0.6911\n", - "[2022-09-15 18:03:42] 3/4, train_loss: 0.6408\n", - "[2022-09-15 18:03:42] 4/4, train_loss: 0.6625\n", - "epoch 1 average loss: 0.6854, best mean dice: -1.0000 at epoch -1\n", - "1 / 4 tensor([[0.2695]], device='cuda:0')\n", - "2 / 4 tensor([[0.2433]], device='cuda:0')\n", - "3 / 4 tensor([[0.2641]], device='cuda:0')\n", - "4 / 4 tensor([[0.3379]], device='cuda:0')\n", - "evaluation metric - class 1: 0.27869710326194763\n", - "avg_metric 0.27869710326194763\n", + "[2022-10-18 08:33:53] 1/4, train_loss: 0.7538\n", + "[2022-10-18 08:33:54] 2/4, train_loss: 0.6982\n", + "[2022-10-18 08:33:54] 3/4, train_loss: 0.7041\n", + "[2022-10-18 08:33:54] 4/4, train_loss: 0.6965\n", + "epoch 1 average loss: 0.7132, best mean dice: -1.0000 at epoch -1\n", + "1 / 4 tensor([[0.3027]], device='cuda:0')\n", + "2 / 4 tensor([[0.3027]], device='cuda:0')\n", + "3 / 4 tensor([[0.3027]], device='cuda:0')\n", + "4 / 4 tensor([[0.3027]], device='cuda:0')\n", + "evaluation metric - class 1: 0.3027001619338989\n", + "avg_metric 0.3027001619338989\n", "saved new best metric model\n", - "current epoch: 1 current mean dice: 0.2787 best mean dice: 0.2787 at epoch 1\n", + "current epoch: 1 current mean dice: 0.3027 best mean dice: 0.3027 at epoch 1\n", "----------\n", "epoch 2/2\n", "learning rate is set to 0.0125\n", - "[2022-09-15 18:03:45] 1/4, train_loss: 0.6572\n", - "[2022-09-15 18:03:46] 2/4, train_loss: 0.6325\n", - "[2022-09-15 18:03:46] 3/4, train_loss: 0.6181\n", - "[2022-09-15 18:03:47] 4/4, train_loss: 0.6344\n", - "epoch 2 average loss: 0.6356, best mean dice: 0.2787 at epoch 1\n", - "1 / 4 tensor([[0.2775]], device='cuda:0')\n", - "2 / 4 tensor([[0.2451]], device='cuda:0')\n", - "3 / 4 tensor([[0.2681]], device='cuda:0')\n", - "4 / 4 tensor([[0.3443]], device='cuda:0')\n", - "evaluation metric - class 1: 0.2837563157081604\n", - "avg_metric 0.2837563157081604\n", + "[2022-10-18 08:33:56] 1/4, train_loss: 0.6745\n", + "[2022-10-18 08:33:56] 2/4, train_loss: 0.6532\n", + "[2022-10-18 08:33:57] 3/4, train_loss: 0.6650\n", + "[2022-10-18 08:33:57] 4/4, train_loss: 0.6286\n", + "epoch 2 average loss: 0.6553, best mean dice: 0.3027 at epoch 1\n", + "1 / 4 tensor([[0.3126]], device='cuda:0')\n", + "2 / 4 tensor([[0.3126]], device='cuda:0')\n", + "3 / 4 tensor([[0.3126]], device='cuda:0')\n", + "4 / 4 tensor([[0.3126]], device='cuda:0')\n", + "evaluation metric - class 1: 0.3125629425048828\n", + "avg_metric 0.3125629425048828\n", "saved new best metric model\n", - "current epoch: 2 current mean dice: 0.2838 best mean dice: 0.2838 at epoch 2\n", - "train completed, best_metric: 0.2838 at epoch: 2\n", - "0.2837563157081604\n", + "current epoch: 2 current mean dice: 0.3126 best mean dice: 0.3126 at epoch 2\n", + "train completed, best_metric: 0.3126 at epoch: 2\n", + "0.3125629425048828\n", "\", stderr=b'')\n", - "2022-09-15 18:03:49,839 - INFO - Launching: python ./workdir/swinunetr_0/scripts/train.py run --config_file='./workdir/swinunetr_0/configs/transforms_train.yaml','./workdir/swinunetr_0/configs/network.yaml','./workdir/swinunetr_0/configs/transforms_validate.yaml','./workdir/swinunetr_0/configs/hyper_parameters.yaml','./workdir/swinunetr_0/configs/transforms_infer.yaml' --num_iterations=8 --num_iterations_per_validation=4 --num_images_per_batch=2 --num_epochs=2 --num_warmup_iterations=4\n", - "2022-09-15 18:04:02,865 - INFO - CompletedProcess(args=['python', './workdir/swinunetr_0/scripts/train.py', 'run', \"--config_file='./workdir/swinunetr_0/configs/transforms_train.yaml','./workdir/swinunetr_0/configs/network.yaml','./workdir/swinunetr_0/configs/transforms_validate.yaml','./workdir/swinunetr_0/configs/hyper_parameters.yaml','./workdir/swinunetr_0/configs/transforms_infer.yaml'\", '--num_iterations=8', '--num_iterations_per_validation=4', '--num_images_per_batch=2', '--num_epochs=2', '--num_warmup_iterations=4'], returncode=0, stdout=b\"[info] number of GPUs: 1\n", + "2022-10-18 08:33:59,001 - INFO - Launching: python ensemble_byoc_work_dir/swinunetr_0/scripts/train.py run --config_file='ensemble_byoc_work_dir/swinunetr_0/configs/transforms_validate.yaml','ensemble_byoc_work_dir/swinunetr_0/configs/transforms_train.yaml','ensemble_byoc_work_dir/swinunetr_0/configs/transforms_infer.yaml','ensemble_byoc_work_dir/swinunetr_0/configs/network.yaml','ensemble_byoc_work_dir/swinunetr_0/configs/hyper_parameters.yaml' --num_iterations=8 --num_iterations_per_validation=4 --num_images_per_batch=2 --num_epochs=2 --num_warmup_iterations=4\n", + "2022-10-18 08:34:06,744 - INFO - CompletedProcess(args=['python', 'ensemble_byoc_work_dir/swinunetr_0/scripts/train.py', 'run', \"--config_file='ensemble_byoc_work_dir/swinunetr_0/configs/transforms_validate.yaml','ensemble_byoc_work_dir/swinunetr_0/configs/transforms_train.yaml','ensemble_byoc_work_dir/swinunetr_0/configs/transforms_infer.yaml','ensemble_byoc_work_dir/swinunetr_0/configs/network.yaml','ensemble_byoc_work_dir/swinunetr_0/configs/hyper_parameters.yaml'\", '--num_iterations=8', '--num_iterations_per_validation=4', '--num_images_per_batch=2', '--num_epochs=2', '--num_warmup_iterations=4'], returncode=0, stdout=b\"[info] number of GPUs: 1\n", "[info] world_size: 1\n", "train_files: 8\n", "val_files: 4\n", @@ -511,40 +504,40 @@ "----------\n", "epoch 1/2\n", "learning rate is set to 0.0001\n", - "[2022-09-15 18:03:57] 1/4, train_loss: 1.2428\n", - "[2022-09-15 18:03:57] 2/4, train_loss: 1.0870\n", - "[2022-09-15 18:03:58] 3/4, train_loss: 1.0164\n", - "[2022-09-15 18:03:58] 4/4, train_loss: 1.0194\n", - "epoch 1 average loss: 1.0914, best mean dice: -1.0000 at epoch -1\n", - "1 / 4 tensor([[0.5926]], device='cuda:0')\n", - "2 / 4 tensor([[0.5186]], device='cuda:0')\n", - "3 / 4 tensor([[0.5887]], device='cuda:0')\n", - "4 / 4 tensor([[0.5910]], device='cuda:0')\n", - "evaluation metric - class 1: 0.5727382898330688\n", - "avg_metric 0.5727382898330688\n", + "[2022-10-18 08:34:03] 1/4, train_loss: 1.3129\n", + "[2022-10-18 08:34:03] 2/4, train_loss: 1.1600\n", + "[2022-10-18 08:34:03] 3/4, train_loss: 1.1060\n", + "[2022-10-18 08:34:04] 4/4, train_loss: 1.0844\n", + "epoch 1 average loss: 1.1658, best mean dice: -1.0000 at epoch -1\n", + "1 / 4 tensor([[0.5112]], device='cuda:0')\n", + "2 / 4 tensor([[0.5112]], device='cuda:0')\n", + "3 / 4 tensor([[0.5112]], device='cuda:0')\n", + "4 / 4 tensor([[0.5112]], device='cuda:0')\n", + "evaluation metric - class 1: 0.5112062096595764\n", + "avg_metric 0.5112062096595764\n", "saved new best metric model\n", - "current epoch: 1 current mean dice: 0.5727 best mean dice: 0.5727 at epoch 1\n", + "current epoch: 1 current mean dice: 0.5112 best mean dice: 0.5112 at epoch 1\n", "----------\n", "epoch 2/2\n", "learning rate is set to 6.25e-06\n", - "[2022-09-15 18:03:59] 1/4, train_loss: 0.9886\n", - "[2022-09-15 18:04:00] 2/4, train_loss: 1.0251\n", - "[2022-09-15 18:04:00] 3/4, train_loss: 0.9945\n", - "[2022-09-15 18:04:00] 4/4, train_loss: 0.9905\n", - "epoch 2 average loss: 0.9997, best mean dice: 0.5727 at epoch 1\n", - "1 / 4 tensor([[0.6000]], device='cuda:0')\n", - "2 / 4 tensor([[0.5266]], device='cuda:0')\n", - "3 / 4 tensor([[0.5965]], device='cuda:0')\n", - "4 / 4 tensor([[0.6011]], device='cuda:0')\n", - "evaluation metric - class 1: 0.5810520052909851\n", - "avg_metric 0.5810520052909851\n", + "[2022-10-18 08:34:05] 1/4, train_loss: 1.0705\n", + "[2022-10-18 08:34:05] 2/4, train_loss: 1.0634\n", + "[2022-10-18 08:34:05] 3/4, train_loss: 1.0623\n", + "[2022-10-18 08:34:05] 4/4, train_loss: 1.0635\n", + "epoch 2 average loss: 1.0649, best mean dice: 0.5112 at epoch 1\n", + "1 / 4 tensor([[0.5264]], device='cuda:0')\n", + "2 / 4 tensor([[0.5264]], device='cuda:0')\n", + "3 / 4 tensor([[0.5264]], device='cuda:0')\n", + "4 / 4 tensor([[0.5264]], device='cuda:0')\n", + "evaluation metric - class 1: 0.5264335870742798\n", + "avg_metric 0.5264335870742798\n", "saved new best metric model\n", - "current epoch: 2 current mean dice: 0.5811 best mean dice: 0.5811 at epoch 2\n", - "train completed, best_metric: 0.5811 at epoch: 2\n", - "0.5810520052909851\n", + "current epoch: 2 current mean dice: 0.5264 best mean dice: 0.5264 at epoch 2\n", + "train completed, best_metric: 0.5264 at epoch: 2\n", + "0.5264335870742798\n", "\", stderr=b'')\n", - "2022-09-15 18:04:02,867 - INFO - Launching: python ./workdir/swinunetr_1/scripts/train.py run --config_file='./workdir/swinunetr_1/configs/transforms_train.yaml','./workdir/swinunetr_1/configs/network.yaml','./workdir/swinunetr_1/configs/transforms_validate.yaml','./workdir/swinunetr_1/configs/hyper_parameters.yaml','./workdir/swinunetr_1/configs/transforms_infer.yaml' --num_iterations=8 --num_iterations_per_validation=4 --num_images_per_batch=2 --num_epochs=2 --num_warmup_iterations=4\n", - "2022-09-15 18:04:15,455 - INFO - CompletedProcess(args=['python', './workdir/swinunetr_1/scripts/train.py', 'run', \"--config_file='./workdir/swinunetr_1/configs/transforms_train.yaml','./workdir/swinunetr_1/configs/network.yaml','./workdir/swinunetr_1/configs/transforms_validate.yaml','./workdir/swinunetr_1/configs/hyper_parameters.yaml','./workdir/swinunetr_1/configs/transforms_infer.yaml'\", '--num_iterations=8', '--num_iterations_per_validation=4', '--num_images_per_batch=2', '--num_epochs=2', '--num_warmup_iterations=4'], returncode=0, stdout=b\"[info] number of GPUs: 1\n", + "2022-10-18 08:34:06,745 - INFO - Launching: python ensemble_byoc_work_dir/swinunetr_1/scripts/train.py run --config_file='ensemble_byoc_work_dir/swinunetr_1/configs/transforms_validate.yaml','ensemble_byoc_work_dir/swinunetr_1/configs/transforms_train.yaml','ensemble_byoc_work_dir/swinunetr_1/configs/transforms_infer.yaml','ensemble_byoc_work_dir/swinunetr_1/configs/network.yaml','ensemble_byoc_work_dir/swinunetr_1/configs/hyper_parameters.yaml' --num_iterations=8 --num_iterations_per_validation=4 --num_images_per_batch=2 --num_epochs=2 --num_warmup_iterations=4\n", + "2022-10-18 08:34:14,326 - INFO - CompletedProcess(args=['python', 'ensemble_byoc_work_dir/swinunetr_1/scripts/train.py', 'run', \"--config_file='ensemble_byoc_work_dir/swinunetr_1/configs/transforms_validate.yaml','ensemble_byoc_work_dir/swinunetr_1/configs/transforms_train.yaml','ensemble_byoc_work_dir/swinunetr_1/configs/transforms_infer.yaml','ensemble_byoc_work_dir/swinunetr_1/configs/network.yaml','ensemble_byoc_work_dir/swinunetr_1/configs/hyper_parameters.yaml'\", '--num_iterations=8', '--num_iterations_per_validation=4', '--num_images_per_batch=2', '--num_epochs=2', '--num_warmup_iterations=4'], returncode=0, stdout=b\"[info] number of GPUs: 1\n", "[info] world_size: 1\n", "train_files: 8\n", "val_files: 4\n", @@ -555,40 +548,40 @@ "----------\n", "epoch 1/2\n", "learning rate is set to 0.0001\n", - "[2022-09-15 18:04:10] 1/4, train_loss: 1.3001\n", - "[2022-09-15 18:04:10] 2/4, train_loss: 1.1246\n", - "[2022-09-15 18:04:10] 3/4, train_loss: 1.0676\n", - "[2022-09-15 18:04:11] 4/4, train_loss: 1.0665\n", - "epoch 1 average loss: 1.1397, best mean dice: -1.0000 at epoch -1\n", - "1 / 4 tensor([[0.4930]], device='cuda:0')\n", - "2 / 4 tensor([[0.4996]], device='cuda:0')\n", - "3 / 4 tensor([[0.4961]], device='cuda:0')\n", - "4 / 4 tensor([[0.5670]], device='cuda:0')\n", - "evaluation metric - class 1: 0.5139241218566895\n", - "avg_metric 0.5139241218566895\n", + "[2022-10-18 08:34:11] 1/4, train_loss: 1.2522\n", + "[2022-10-18 08:34:11] 2/4, train_loss: 1.0597\n", + "[2022-10-18 08:34:11] 3/4, train_loss: 0.9993\n", + "[2022-10-18 08:34:11] 4/4, train_loss: 0.9734\n", + "epoch 1 average loss: 1.0712, best mean dice: -1.0000 at epoch -1\n", + "1 / 4 tensor([[0.5725]], device='cuda:0')\n", + "2 / 4 tensor([[0.5725]], device='cuda:0')\n", + "3 / 4 tensor([[0.5725]], device='cuda:0')\n", + "4 / 4 tensor([[0.5725]], device='cuda:0')\n", + "evaluation metric - class 1: 0.5725038051605225\n", + "avg_metric 0.5725038051605225\n", "saved new best metric model\n", - "current epoch: 1 current mean dice: 0.5139 best mean dice: 0.5139 at epoch 1\n", + "current epoch: 1 current mean dice: 0.5725 best mean dice: 0.5725 at epoch 1\n", "----------\n", "epoch 2/2\n", "learning rate is set to 6.25e-06\n", - "[2022-09-15 18:04:12] 1/4, train_loss: 1.0474\n", - "[2022-09-15 18:04:12] 2/4, train_loss: 1.0310\n", - "[2022-09-15 18:04:13] 3/4, train_loss: 1.0413\n", - "[2022-09-15 18:04:13] 4/4, train_loss: 1.0121\n", - "epoch 2 average loss: 1.0330, best mean dice: 0.5139 at epoch 1\n", - "1 / 4 tensor([[0.5099]], device='cuda:0')\n", - "2 / 4 tensor([[0.5167]], device='cuda:0')\n", - "3 / 4 tensor([[0.5185]], device='cuda:0')\n", - "4 / 4 tensor([[0.5839]], device='cuda:0')\n", - "evaluation metric - class 1: 0.5322240591049194\n", - "avg_metric 0.5322240591049194\n", + "[2022-10-18 08:34:12] 1/4, train_loss: 0.9589\n", + "[2022-10-18 08:34:12] 2/4, train_loss: 0.9553\n", + "[2022-10-18 08:34:12] 3/4, train_loss: 0.9514\n", + "[2022-10-18 08:34:13] 4/4, train_loss: 0.9499\n", + "epoch 2 average loss: 0.9539, best mean dice: 0.5725 at epoch 1\n", + "1 / 4 tensor([[0.5843]], device='cuda:0')\n", + "2 / 4 tensor([[0.5843]], device='cuda:0')\n", + "3 / 4 tensor([[0.5843]], device='cuda:0')\n", + "4 / 4 tensor([[0.5843]], device='cuda:0')\n", + "evaluation metric - class 1: 0.5842882394790649\n", + "avg_metric 0.5842882394790649\n", "saved new best metric model\n", - "current epoch: 2 current mean dice: 0.5322 best mean dice: 0.5322 at epoch 2\n", - "train completed, best_metric: 0.5322 at epoch: 2\n", - "0.5322240591049194\n", + "current epoch: 2 current mean dice: 0.5843 best mean dice: 0.5843 at epoch 2\n", + "train completed, best_metric: 0.5843 at epoch: 2\n", + "0.5842882394790649\n", "\", stderr=b'')\n", - "2022-09-15 18:04:15,457 - INFO - Launching: python ./workdir/segresnet_0/scripts/train.py run --config_file='./workdir/segresnet_0/configs/transforms_train.yaml','./workdir/segresnet_0/configs/network.yaml','./workdir/segresnet_0/configs/transforms_validate.yaml','./workdir/segresnet_0/configs/hyper_parameters.yaml','./workdir/segresnet_0/configs/transforms_infer.yaml' --num_iterations=8 --num_iterations_per_validation=4 --num_images_per_batch=2 --num_epochs=2 --num_warmup_iterations=4\n", - "2022-09-15 18:04:26,638 - INFO - CompletedProcess(args=['python', './workdir/segresnet_0/scripts/train.py', 'run', \"--config_file='./workdir/segresnet_0/configs/transforms_train.yaml','./workdir/segresnet_0/configs/network.yaml','./workdir/segresnet_0/configs/transforms_validate.yaml','./workdir/segresnet_0/configs/hyper_parameters.yaml','./workdir/segresnet_0/configs/transforms_infer.yaml'\", '--num_iterations=8', '--num_iterations_per_validation=4', '--num_images_per_batch=2', '--num_epochs=2', '--num_warmup_iterations=4'], returncode=0, stdout=b\"[info] number of GPUs: 1\n", + "2022-10-18 08:34:14,327 - INFO - Launching: python ensemble_byoc_work_dir/segresnet_0/scripts/train.py run --config_file='ensemble_byoc_work_dir/segresnet_0/configs/transforms_validate.yaml','ensemble_byoc_work_dir/segresnet_0/configs/transforms_train.yaml','ensemble_byoc_work_dir/segresnet_0/configs/transforms_infer.yaml','ensemble_byoc_work_dir/segresnet_0/configs/network.yaml','ensemble_byoc_work_dir/segresnet_0/configs/hyper_parameters.yaml' --num_iterations=8 --num_iterations_per_validation=4 --num_images_per_batch=2 --num_epochs=2 --num_warmup_iterations=4\n", + "2022-10-18 08:34:21,171 - INFO - CompletedProcess(args=['python', 'ensemble_byoc_work_dir/segresnet_0/scripts/train.py', 'run', \"--config_file='ensemble_byoc_work_dir/segresnet_0/configs/transforms_validate.yaml','ensemble_byoc_work_dir/segresnet_0/configs/transforms_train.yaml','ensemble_byoc_work_dir/segresnet_0/configs/transforms_infer.yaml','ensemble_byoc_work_dir/segresnet_0/configs/network.yaml','ensemble_byoc_work_dir/segresnet_0/configs/hyper_parameters.yaml'\", '--num_iterations=8', '--num_iterations_per_validation=4', '--num_images_per_batch=2', '--num_epochs=2', '--num_warmup_iterations=4'], returncode=0, stdout=b\"[info] number of GPUs: 1\n", "[info] world_size: 1\n", "train_files: 8\n", "val_files: 4\n", @@ -599,40 +592,40 @@ "----------\n", "epoch 1/2\n", "learning rate is set to 2e-05\n", - "[2022-09-15 18:04:23] 1/4, train_loss: 1.2420\n", - "[2022-09-15 18:04:23] 2/4, train_loss: 1.1942\n", - "[2022-09-15 18:04:23] 3/4, train_loss: 1.1724\n", - "[2022-09-15 18:04:23] 4/4, train_loss: 1.1518\n", - "epoch 1 average loss: 1.1901, best mean dice: -1.0000 at epoch -1\n", - "1 / 4 tensor([[0.5260]], device='cuda:0')\n", - "2 / 4 tensor([[0.4520]], device='cuda:0')\n", - "3 / 4 tensor([[0.5202]], device='cuda:0')\n", - "4 / 4 tensor([[0.5253]], device='cuda:0')\n", - "evaluation metric - class 1: 0.5058972239494324\n", - "avg_metric 0.5058972239494324\n", + "[2022-10-18 08:34:18] 1/4, train_loss: 1.0918\n", + "[2022-10-18 08:34:18] 2/4, train_loss: 1.0556\n", + "[2022-10-18 08:34:19] 3/4, train_loss: 0.8768\n", + "[2022-10-18 08:34:19] 4/4, train_loss: 0.8789\n", + "epoch 1 average loss: 0.9758, best mean dice: -1.0000 at epoch -1\n", + "1 / 4 tensor([[0.2607]], device='cuda:0')\n", + "2 / 4 tensor([[0.2607]], device='cuda:0')\n", + "3 / 4 tensor([[0.2607]], device='cuda:0')\n", + "4 / 4 tensor([[0.2607]], device='cuda:0')\n", + "evaluation metric - class 1: 0.26070746779441833\n", + "avg_metric 0.26070746779441833\n", "saved new best metric model\n", - "current epoch: 1 current mean dice: 0.5059 best mean dice: 0.5059 at epoch 1\n", + "current epoch: 1 current mean dice: 0.2607 best mean dice: 0.2607 at epoch 1\n", "----------\n", "epoch 2/2\n", "learning rate is set to 0.0\n", - "[2022-09-15 18:04:24] 1/4, train_loss: 1.1045\n", - "[2022-09-15 18:04:24] 2/4, train_loss: 1.0873\n", - "[2022-09-15 18:04:24] 3/4, train_loss: 1.0731\n", - "[2022-09-15 18:04:24] 4/4, train_loss: 1.0609\n", - "epoch 2 average loss: 1.0815, best mean dice: 0.5059 at epoch 1\n", - "1 / 4 tensor([[0.6442]], device='cuda:0')\n", - "2 / 4 tensor([[0.5856]], device='cuda:0')\n", - "3 / 4 tensor([[0.6486]], device='cuda:0')\n", - "4 / 4 tensor([[0.6706]], device='cuda:0')\n", - "evaluation metric - class 1: 0.6372344493865967\n", - "avg_metric 0.6372344493865967\n", + "[2022-10-18 08:34:19] 1/4, train_loss: 0.7986\n", + "[2022-10-18 08:34:19] 2/4, train_loss: 0.8023\n", + "[2022-10-18 08:34:20] 3/4, train_loss: 0.7758\n", + "[2022-10-18 08:34:20] 4/4, train_loss: 0.7784\n", + "epoch 2 average loss: 0.7888, best mean dice: 0.2607 at epoch 1\n", + "1 / 4 tensor([[0.4514]], device='cuda:0')\n", + "2 / 4 tensor([[0.4514]], device='cuda:0')\n", + "3 / 4 tensor([[0.4514]], device='cuda:0')\n", + "4 / 4 tensor([[0.4514]], device='cuda:0')\n", + "evaluation metric - class 1: 0.4513593018054962\n", + "avg_metric 0.4513593018054962\n", "saved new best metric model\n", - "current epoch: 2 current mean dice: 0.6372 best mean dice: 0.6372 at epoch 2\n", - "train completed, best_metric: 0.6372 at epoch: 2\n", - "0.6372344493865967\n", + "current epoch: 2 current mean dice: 0.4514 best mean dice: 0.4514 at epoch 2\n", + "train completed, best_metric: 0.4514 at epoch: 2\n", + "0.4513593018054962\n", "\", stderr=b'')\n", - "2022-09-15 18:04:26,639 - INFO - Launching: python ./workdir/segresnet_1/scripts/train.py run --config_file='./workdir/segresnet_1/configs/transforms_train.yaml','./workdir/segresnet_1/configs/network.yaml','./workdir/segresnet_1/configs/transforms_validate.yaml','./workdir/segresnet_1/configs/hyper_parameters.yaml','./workdir/segresnet_1/configs/transforms_infer.yaml' --num_iterations=8 --num_iterations_per_validation=4 --num_images_per_batch=2 --num_epochs=2 --num_warmup_iterations=4\n", - "2022-09-15 18:04:37,901 - INFO - CompletedProcess(args=['python', './workdir/segresnet_1/scripts/train.py', 'run', \"--config_file='./workdir/segresnet_1/configs/transforms_train.yaml','./workdir/segresnet_1/configs/network.yaml','./workdir/segresnet_1/configs/transforms_validate.yaml','./workdir/segresnet_1/configs/hyper_parameters.yaml','./workdir/segresnet_1/configs/transforms_infer.yaml'\", '--num_iterations=8', '--num_iterations_per_validation=4', '--num_images_per_batch=2', '--num_epochs=2', '--num_warmup_iterations=4'], returncode=0, stdout=b\"[info] number of GPUs: 1\n", + "2022-10-18 08:34:21,172 - INFO - Launching: python ensemble_byoc_work_dir/segresnet_1/scripts/train.py run --config_file='ensemble_byoc_work_dir/segresnet_1/configs/transforms_validate.yaml','ensemble_byoc_work_dir/segresnet_1/configs/transforms_train.yaml','ensemble_byoc_work_dir/segresnet_1/configs/transforms_infer.yaml','ensemble_byoc_work_dir/segresnet_1/configs/network.yaml','ensemble_byoc_work_dir/segresnet_1/configs/hyper_parameters.yaml' --num_iterations=8 --num_iterations_per_validation=4 --num_images_per_batch=2 --num_epochs=2 --num_warmup_iterations=4\n", + "2022-10-18 08:34:27,921 - INFO - CompletedProcess(args=['python', 'ensemble_byoc_work_dir/segresnet_1/scripts/train.py', 'run', \"--config_file='ensemble_byoc_work_dir/segresnet_1/configs/transforms_validate.yaml','ensemble_byoc_work_dir/segresnet_1/configs/transforms_train.yaml','ensemble_byoc_work_dir/segresnet_1/configs/transforms_infer.yaml','ensemble_byoc_work_dir/segresnet_1/configs/network.yaml','ensemble_byoc_work_dir/segresnet_1/configs/hyper_parameters.yaml'\", '--num_iterations=8', '--num_iterations_per_validation=4', '--num_images_per_batch=2', '--num_epochs=2', '--num_warmup_iterations=4'], returncode=0, stdout=b\"[info] number of GPUs: 1\n", "[info] world_size: 1\n", "train_files: 8\n", "val_files: 4\n", @@ -643,37 +636,37 @@ "----------\n", "epoch 1/2\n", "learning rate is set to 2e-05\n", - "[2022-09-15 18:04:34] 1/4, train_loss: 1.4037\n", - "[2022-09-15 18:04:34] 2/4, train_loss: 1.4404\n", - "[2022-09-15 18:04:34] 3/4, train_loss: 1.3031\n", - "[2022-09-15 18:04:34] 4/4, train_loss: 1.2646\n", - "epoch 1 average loss: 1.3529, best mean dice: -1.0000 at epoch -1\n", - "1 / 4 tensor([[0.2550]], device='cuda:0')\n", - "2 / 4 tensor([[0.2621]], device='cuda:0')\n", - "3 / 4 tensor([[0.1281]], device='cuda:0')\n", - "4 / 4 tensor([[0.3087]], device='cuda:0')\n", - "evaluation metric - class 1: 0.23847287893295288\n", - "avg_metric 0.23847287893295288\n", + "[2022-10-18 08:34:25] 1/4, train_loss: 1.0369\n", + "[2022-10-18 08:34:25] 2/4, train_loss: 0.9921\n", + "[2022-10-18 08:34:25] 3/4, train_loss: 0.8756\n", + "[2022-10-18 08:34:25] 4/4, train_loss: 0.8824\n", + "epoch 1 average loss: 0.9468, best mean dice: -1.0000 at epoch -1\n", + "1 / 4 tensor([[0.3747]], device='cuda:0')\n", + "2 / 4 tensor([[0.3747]], device='cuda:0')\n", + "3 / 4 tensor([[0.3747]], device='cuda:0')\n", + "4 / 4 tensor([[0.3747]], device='cuda:0')\n", + "evaluation metric - class 1: 0.3747006356716156\n", + "avg_metric 0.3747006356716156\n", "saved new best metric model\n", - "current epoch: 1 current mean dice: 0.2385 best mean dice: 0.2385 at epoch 1\n", + "current epoch: 1 current mean dice: 0.3747 best mean dice: 0.3747 at epoch 1\n", "----------\n", "epoch 2/2\n", "learning rate is set to 0.0\n", - "[2022-09-15 18:04:35] 1/4, train_loss: 1.2191\n", - "[2022-09-15 18:04:35] 2/4, train_loss: 1.2035\n", - "[2022-09-15 18:04:36] 3/4, train_loss: 1.2202\n", - "[2022-09-15 18:04:36] 4/4, train_loss: 1.1515\n", - "epoch 2 average loss: 1.1986, best mean dice: 0.2385 at epoch 1\n", - "1 / 4 tensor([[0.4515]], device='cuda:0')\n", - "2 / 4 tensor([[0.4553]], device='cuda:0')\n", - "3 / 4 tensor([[0.4585]], device='cuda:0')\n", - "4 / 4 tensor([[0.5423]], device='cuda:0')\n", - "evaluation metric - class 1: 0.47688400745391846\n", - "avg_metric 0.47688400745391846\n", + "[2022-10-18 08:34:26] 1/4, train_loss: 0.8145\n", + "[2022-10-18 08:34:26] 2/4, train_loss: 0.8149\n", + "[2022-10-18 08:34:26] 3/4, train_loss: 0.7835\n", + "[2022-10-18 08:34:26] 4/4, train_loss: 0.7795\n", + "epoch 2 average loss: 0.7981, best mean dice: 0.3747 at epoch 1\n", + "1 / 4 tensor([[0.4830]], device='cuda:0')\n", + "2 / 4 tensor([[0.4830]], device='cuda:0')\n", + "3 / 4 tensor([[0.4830]], device='cuda:0')\n", + "4 / 4 tensor([[0.4830]], device='cuda:0')\n", + "evaluation metric - class 1: 0.48304688930511475\n", + "avg_metric 0.48304688930511475\n", "saved new best metric model\n", - "current epoch: 2 current mean dice: 0.4769 best mean dice: 0.4769 at epoch 2\n", - "train completed, best_metric: 0.4769 at epoch: 2\n", - "0.47688400745391846\n", + "current epoch: 2 current mean dice: 0.4830 best mean dice: 0.4830 at epoch 2\n", + "train completed, best_metric: 0.4830 at epoch: 2\n", + "0.48304688930511475\n", "\", stderr=b'')\n" ] } @@ -725,7 +718,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 11, "metadata": {}, "outputs": [ { @@ -733,23 +726,23 @@ "output_type": "stream", "text": [ "0\n", - "[info] checkpoint ./workdir/segresnet_0/model_fold0/best_metric_model.pt loaded\n", - "2022-09-15 18:05:59,870 INFO image_writer.py:194 - writing: workdir/segresnet_0/prediction_testing/val_001.fake/val_001.fake_seg.nii.gz\n", - "[info] checkpoint ./workdir/swinunetr_0/model_fold0/best_metric_model.pt loaded\n", - "2022-09-15 18:06:00,831 INFO image_writer.py:194 - writing: workdir/swinunetr_0/prediction_testing/val_001.fake/val_001.fake_seg.nii.gz\n", - "[info] checkpoint ./workdir/dints_0/model_fold0/best_metric_model.pt loaded\n", - "2022-09-15 18:06:03,386 INFO image_writer.py:194 - writing: workdir/dints_0/prediction_testing/val_001.fake/val_001.fake_seg.nii.gz\n", + "[info] checkpoint ensemble_byoc_work_dir/segresnet2d_0/model_fold0/best_metric_model.pt loaded\n", + "2022-10-18 08:34:29,454 INFO image_writer.py:193 - writing: ensemble_byoc_work_dir/segresnet2d_0/prediction_testing/val_001.fake/val_001.fake_seg.nii.gz\n", + "[info] checkpoint ensemble_byoc_work_dir/segresnet2d_1/model_fold1/best_metric_model.pt loaded\n", + "2022-10-18 08:34:29,806 INFO image_writer.py:193 - writing: ensemble_byoc_work_dir/segresnet2d_1/prediction_testing/val_001.fake/val_001.fake_seg.nii.gz\n", + "[info] checkpoint ensemble_byoc_work_dir/swinunetr_1/model_fold1/best_metric_model.pt loaded\n", + "2022-10-18 08:34:30,432 INFO image_writer.py:193 - writing: ensemble_byoc_work_dir/swinunetr_1/prediction_testing/val_001.fake/val_001.fake_seg.nii.gz\n", "1\n", - "[info] checkpoint ./workdir/segresnet_0/model_fold0/best_metric_model.pt loaded\n", - "2022-09-15 18:06:04,433 INFO image_writer.py:194 - writing: workdir/segresnet_0/prediction_testing/val_002.fake/val_002.fake_seg.nii.gz\n", - "[info] checkpoint ./workdir/swinunetr_0/model_fold0/best_metric_model.pt loaded\n", - "2022-09-15 18:06:05,261 INFO image_writer.py:194 - writing: workdir/swinunetr_0/prediction_testing/val_002.fake/val_002.fake_seg.nii.gz\n", - "[info] checkpoint ./workdir/dints_0/model_fold0/best_metric_model.pt loaded\n", - "2022-09-15 18:06:07,385 INFO image_writer.py:194 - writing: workdir/dints_0/prediction_testing/val_002.fake/val_002.fake_seg.nii.gz\n", + "[info] checkpoint ensemble_byoc_work_dir/segresnet2d_0/model_fold0/best_metric_model.pt loaded\n", + "2022-10-18 08:34:30,880 INFO image_writer.py:193 - writing: ensemble_byoc_work_dir/segresnet2d_0/prediction_testing/val_002.fake/val_002.fake_seg.nii.gz\n", + "[info] checkpoint ensemble_byoc_work_dir/segresnet2d_1/model_fold1/best_metric_model.pt loaded\n", + "2022-10-18 08:34:31,281 INFO image_writer.py:193 - writing: ensemble_byoc_work_dir/segresnet2d_1/prediction_testing/val_002.fake/val_002.fake_seg.nii.gz\n", + "[info] checkpoint ensemble_byoc_work_dir/swinunetr_1/model_fold1/best_metric_model.pt loaded\n", + "2022-10-18 08:34:31,887 INFO image_writer.py:193 - writing: ensemble_byoc_work_dir/swinunetr_1/prediction_testing/val_002.fake/val_002.fake_seg.nii.gz\n", "The ensemble randomly picks the following models:\n", - "segresnet_0\n", - "swinunetr_0\n", - "dints_0\n" + "segresnet2d_0\n", + "segresnet2d_1\n", + "swinunetr_1\n" ] } ], diff --git a/auto3dseg/notebooks/hpo_nni.ipynb b/auto3dseg/notebooks/hpo_nni.ipynb index 4f3bb484e5..0420ca85de 100644 --- a/auto3dseg/notebooks/hpo_nni.ipynb +++ b/auto3dseg/notebooks/hpo_nni.ipynb @@ -6,7 +6,7 @@ "source": [ "# MONAI Auto3DSeg Hyper-parameter Optimization with NNI\n", "\n", - "This notebook provides an example to perform hype-parameter optimization(HPO) on learning rate with grid search method for spleen segmentation using NNI.\n", + "This notebook provides an example to perform hype-parameter optimization(HPO) on learning rate with grid search method for hippocampus segmentation using NNI.\n", "\n", "Note: if you have used other notebooks under `auto3dseg`, for examples: \n", "- `auto_runner.ipynb`\n", @@ -45,6 +45,8 @@ "import torch\n", "import yaml\n", "\n", + "from pathlib import Path\n", + "\n", "# download MSD dataset\n", "from monai.apps import download_and_extract\n", "from monai.apps.auto3dseg import BundleGen, DataAnalyzer, NNIGen\n", @@ -66,8 +68,8 @@ "outputs": [], "source": [ "# Dataset pathes\n", - "data_root = \"./\"\n", - "msd_task = \"Task09_Spleen\"\n", + "data_root = str(Path(\".\"))\n", + "msd_task = \"Task04_Hippocampus\"\n", "dataroot = os.path.join(data_root, msd_task)\n", "\n", "# User created files\n", diff --git a/auto3dseg/notebooks/hpo_optuna.ipynb b/auto3dseg/notebooks/hpo_optuna.ipynb index 48e7b589ed..9f5c0c3746 100644 --- a/auto3dseg/notebooks/hpo_optuna.ipynb +++ b/auto3dseg/notebooks/hpo_optuna.ipynb @@ -6,7 +6,7 @@ "source": [ "# MONAI Auto3DSeg Hyper-parameter Optimization with Optuna\n", "\n", - "This notebook provides an example to perform hype-parameter optimization(HPO) on learning rate with grid search method for spleen segmentation using Optuna.\n", + "This notebook provides an example to perform hype-parameter optimization(HPO) on learning rate with grid search method for hippocampus segmentation using Optuna.\n", "\n", "Note: if you have used other notebooks under `auto3dseg`, for examples: \n", "- `auto_runner.ipynb`\n", @@ -47,6 +47,7 @@ "import yaml\n", "\n", "from functools import partial\n", + "from pathlib import Path\n", "\n", "from monai.apps import download_and_extract\n", "from monai.apps.auto3dseg import BundleGen, DataAnalyzer, OptunaGen\n", @@ -68,8 +69,8 @@ "outputs": [], "source": [ "# Dataset pathes\n", - "data_root = \"./\"\n", - "msd_task = \"Task09_Spleen\"\n", + "data_root = str(Path(\".\"))\n", + "msd_task = \"Task04_Hippocampus\"\n", "dataroot = os.path.join(data_root, msd_task)\n", "\n", "# User created files\n", @@ -152,31 +153,17 @@ "name": "stderr", "output_type": "stream", "text": [ - "100%|██████████| 40/40 [01:33<00:00, 2.34s/it]" + "100%|██████████| 30/30 [00:00<00:00, 79.05it/s]\n", + "algo_templates.tar.gz: 296kB [00:01, 164kB/s] " ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2022-09-28 17:08:51,151 - WARNING - Data is not completely uniform. MONAI transforms may provide unexpected result\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n", - "algo_templates.tar.gz: 100%|██████████| 280k/280k [00:00<00:00, 341kB/s] " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2022-09-28 17:08:52,165 - INFO - Downloaded: /tmp/tmpwptjcv79/algo_templates.tar.gz\n", - "2022-09-28 17:08:52,165 - INFO - Expected md5 is None, skip md5 check for file /tmp/tmpwptjcv79/algo_templates.tar.gz.\n", - "2022-09-28 17:08:52,166 - INFO - Writing into directory: ./hpo_optuna_work_dir.\n" + "2022-10-18 08:43:01,671 - INFO - Downloaded: /tmp/tmpapjtatl0/algo_templates.tar.gz\n", + "2022-10-18 08:43:01,673 - INFO - Expected md5 is None, skip md5 check for file /tmp/tmpapjtatl0/algo_templates.tar.gz.\n", + "2022-10-18 08:43:01,676 - INFO - Writing into directory: ./hpo_optuna_work_dir.\n" ] }, { @@ -190,26 +177,26 @@ "name": "stdout", "output_type": "stream", "text": [ - "2022-09-28 17:08:59,605 - INFO - ./hpo_optuna_work_dir/segresnet2d_0\n", - "2022-09-28 17:09:01,295 - INFO - ./hpo_optuna_work_dir/segresnet2d_1\n", - "2022-09-28 17:09:03,774 - INFO - ./hpo_optuna_work_dir/segresnet2d_2\n", - "2022-09-28 17:09:04,496 - INFO - ./hpo_optuna_work_dir/segresnet2d_3\n", - "2022-09-28 17:09:05,354 - INFO - ./hpo_optuna_work_dir/segresnet2d_4\n", - "2022-09-28 17:09:06,206 - INFO - ./hpo_optuna_work_dir/dints_0\n", - "2022-09-28 17:09:07,010 - INFO - ./hpo_optuna_work_dir/dints_1\n", - "2022-09-28 17:09:09,408 - INFO - ./hpo_optuna_work_dir/dints_2\n", - "2022-09-28 17:09:12,231 - INFO - ./hpo_optuna_work_dir/dints_3\n", - "2022-09-28 17:09:14,473 - INFO - ./hpo_optuna_work_dir/dints_4\n", - "2022-09-28 17:09:15,362 - INFO - ./hpo_optuna_work_dir/swinunetr_0\n", - "2022-09-28 17:09:16,713 - INFO - ./hpo_optuna_work_dir/swinunetr_1\n", - "2022-09-28 17:09:20,002 - INFO - ./hpo_optuna_work_dir/swinunetr_2\n", - "2022-09-28 17:09:21,317 - INFO - ./hpo_optuna_work_dir/swinunetr_3\n", - "2022-09-28 17:09:23,820 - INFO - ./hpo_optuna_work_dir/swinunetr_4\n", - "2022-09-28 17:09:25,380 - INFO - ./hpo_optuna_work_dir/segresnet_0\n", - "2022-09-28 17:09:27,202 - INFO - ./hpo_optuna_work_dir/segresnet_1\n", - "2022-09-28 17:09:29,275 - INFO - ./hpo_optuna_work_dir/segresnet_2\n", - "2022-09-28 17:09:30,899 - INFO - ./hpo_optuna_work_dir/segresnet_3\n", - "2022-09-28 17:09:32,433 - INFO - ./hpo_optuna_work_dir/segresnet_4\n" + "2022-10-18 08:43:01,997 - INFO - ./hpo_optuna_work_dir/segresnet2d_0\n", + "2022-10-18 08:43:02,252 - INFO - ./hpo_optuna_work_dir/segresnet2d_1\n", + "2022-10-18 08:43:02,622 - INFO - ./hpo_optuna_work_dir/segresnet2d_2\n", + "2022-10-18 08:43:02,896 - INFO - ./hpo_optuna_work_dir/segresnet2d_3\n", + "2022-10-18 08:43:03,165 - INFO - ./hpo_optuna_work_dir/segresnet2d_4\n", + "2022-10-18 08:43:03,441 - INFO - ./hpo_optuna_work_dir/dints_0\n", + "2022-10-18 08:43:03,819 - INFO - ./hpo_optuna_work_dir/dints_1\n", + "2022-10-18 08:43:04,092 - INFO - ./hpo_optuna_work_dir/dints_2\n", + "2022-10-18 08:43:04,375 - INFO - ./hpo_optuna_work_dir/dints_3\n", + "2022-10-18 08:43:04,760 - INFO - ./hpo_optuna_work_dir/dints_4\n", + "2022-10-18 08:43:05,023 - INFO - ./hpo_optuna_work_dir/swinunetr_0\n", + "2022-10-18 08:43:05,293 - INFO - ./hpo_optuna_work_dir/swinunetr_1\n", + "2022-10-18 08:43:05,559 - INFO - ./hpo_optuna_work_dir/swinunetr_2\n", + "2022-10-18 08:43:05,913 - INFO - ./hpo_optuna_work_dir/swinunetr_3\n", + "2022-10-18 08:43:06,167 - INFO - ./hpo_optuna_work_dir/swinunetr_4\n", + "2022-10-18 08:43:06,450 - INFO - ./hpo_optuna_work_dir/segresnet_0\n", + "2022-10-18 08:43:06,726 - INFO - ./hpo_optuna_work_dir/segresnet_1\n", + "2022-10-18 08:43:07,140 - INFO - ./hpo_optuna_work_dir/segresnet_2\n", + "2022-10-18 08:43:07,413 - INFO - ./hpo_optuna_work_dir/segresnet_3\n", + "2022-10-18 08:43:07,693 - INFO - ./hpo_optuna_work_dir/segresnet_4\n" ] } ], @@ -306,7 +293,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "2022-09-28 17:09:37,228 - INFO - ./hpo_optuna_work_dir/segresnet2d_0_override\n" + "2022-10-18 08:43:08,171 - INFO - ./hpo_optuna_work_dir/segresnet2d_0_override\n" ] } ], @@ -335,192 +322,185 @@ "name": "stderr", "output_type": "stream", "text": [ - "\u001b[32m[I 2022-09-28 17:09:37,486]\u001b[0m A new study created in memory with name: no-name-a34b729c-9f06-4439-b528-5199836c43d3\u001b[0m\n" + "\u001b[32m[I 2022-10-18 08:43:08,301]\u001b[0m A new study created in memory with name: no-name-2c662b87-ae33-42e4-a7a2-6ec258d89b5a\u001b[0m\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2022-09-28 17:09:39,227 - INFO - ./optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.001\n", - "2022-09-28 17:09:39,241 - INFO - Launching: python ./optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.001/scripts/train.py run --config_file='./optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.001/configs/network.yaml','./optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.001/configs/transforms_validate.yaml','./optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.001/configs/transforms_infer.yaml','./optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.001/configs/hyper_parameters.yaml','./optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.001/configs/transforms_train.yaml' --learning_rate=0.001\n", - "2022-09-28 17:12:50,088 - INFO - CompletedProcess(args=['python', './optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.001/scripts/train.py', 'run', \"--config_file='./optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.001/configs/network.yaml','./optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.001/configs/transforms_validate.yaml','./optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.001/configs/transforms_infer.yaml','./optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.001/configs/hyper_parameters.yaml','./optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.001/configs/transforms_train.yaml'\", '--learning_rate=0.001'], returncode=0, stdout=b\"[info] number of GPUs: 1\n", - "[info] world_size: 1\n", - "train_files: 32\n", - "val_files: 8\n", + "2022-10-18 08:43:08,341 - INFO - ./optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.1\n", + "2022-10-18 08:43:08,341 - INFO - Launching: torchrun --nnodes=1 --nproc_per_node=2 ./optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.1/scripts/train.py run --config_file='optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.1/configs/transforms_validate.yaml','optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.1/configs/transforms_train.yaml','optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.1/configs/transforms_infer.yaml','optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.1/configs/network.yaml','optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.1/configs/hyper_parameters.yaml' --learning_rate=0.1\n", + "2022-10-18 08:43:19,047 - INFO - CompletedProcess(args=['torchrun', '--nnodes=1', '--nproc_per_node=2', './optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.1/scripts/train.py', 'run', \"--config_file='optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.1/configs/transforms_validate.yaml','optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.1/configs/transforms_train.yaml','optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.1/configs/transforms_infer.yaml','optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.1/configs/network.yaml','optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.1/configs/hyper_parameters.yaml'\", '--learning_rate=0.1'], returncode=0, stdout=b\"[info] number of GPUs: 2\n", + "2022-10-18 08:43:11,691 - Added key: store_based_barrier_key:1 to store for rank: 1\n", + "[info] number of GPUs: 2\n", + "2022-10-18 08:43:11,825 - Added key: store_based_barrier_key:1 to store for rank: 0\n", + "2022-10-18 08:43:11,825 - Rank 0: Completed store-based barrier for key:store_based_barrier_key:1 with 2 nodes.\n", + "[info] world_size: 2\n", + "train_files: 12\n", + "val_files: 3\n", + "2022-10-18 08:43:11,834 - Rank 1: Completed store-based barrier for key:store_based_barrier_key:1 with 2 nodes.\n", + "[info] world_size: 2\n", + "train_files: 12\n", + "val_files: 3\n", "num_epochs 2\n", "num_epochs_per_validation 1\n", - "[info] training from scratch\n", + "[info] training from scratch[info] training from scratch\n", + "\n", "[info] amp enabled\n", "----------\n", "epoch 1/2\n", - "learning rate is set to 0.001\n", - "[2022-09-28 17:10:21] 1/16, train_loss: 0.8308\n", - "[2022-09-28 17:10:21] 2/16, train_loss: 0.8357\n", - "[2022-09-28 17:10:21] 3/16, train_loss: 0.7409\n", - "[2022-09-28 17:10:21] 4/16, train_loss: 0.8472\n", - "[2022-09-28 17:10:21] 5/16, train_loss: 0.7716\n", - "[2022-09-28 17:10:21] 6/16, train_loss: 0.8473\n", - "[2022-09-28 17:10:21] 7/16, train_loss: 0.8624\n", - "[2022-09-28 17:10:21] 8/16, train_loss: 0.7660\n", - "[2022-09-28 17:10:27] 9/16, train_loss: 0.7231\n", - "[2022-09-28 17:10:27] 10/16, train_loss: 0.8150\n", - "[2022-09-28 17:10:27] 11/16, train_loss: 0.8157\n", - "[2022-09-28 17:10:27] 12/16, train_loss: 0.7737\n", - "[2022-09-28 17:10:27] 13/16, train_loss: 0.7800\n", - "[2022-09-28 17:10:27] 14/16, train_loss: 0.7598\n", - "[2022-09-28 17:10:31] 15/16, train_loss: 0.7847\n", - "[2022-09-28 17:10:31] 16/16, train_loss: 0.7580\n", - "epoch 1 average loss: 0.7945, best mean dice: -1.0000 at epoch -1\n", - "1 / 8 tensor([[0.0232]], device='cuda:0')\n", - "2 / 8 tensor([[0.0230]], device='cuda:0')\n", - "3 / 8 tensor([[0.0052]], device='cuda:0')\n", - "4 / 8 tensor([[0.0066]], device='cuda:0')\n", - "5 / 8 tensor([[0.0135]], device='cuda:0')\n", - "6 / 8 tensor([[0.0088]], device='cuda:0')\n", - "7 / 8 tensor([[0.0101]], device='cuda:0')\n", - "8 / 8 tensor([[0.0118]], device='cuda:0')\n", - "evaluation metric - class 1: 0.012783133424818516\n", - "avg_metric 0.012783133424818516\n", + "learning rate is set to 0.1\n", + "[2022-10-18 08:43:14] 1/6, train_loss: 1.0068\n", + "2022-10-18 08:43:14,298 - Reducer buckets have been rebuilt in this iteration.\n", + "2022-10-18 08:43:14,298 - Reducer buckets have been rebuilt in this iteration.\n", + "[2022-10-18 08:43:14] 2/6, train_loss: 0.8218\n", + "[2022-10-18 08:43:14] 3/6, train_loss: 0.7328\n", + "[2022-10-18 08:43:14] 4/6, train_loss: 0.7411\n", + "[2022-10-18 08:43:14] 5/6, train_loss: 0.6738\n", + "[2022-10-18 08:43:14] 6/6, train_loss: 0.7053\n", + "epoch 1 average loss: 0.8018, best mean dice: -1.0000 at epoch -1\n", + "1 / 3 tensor([[0., 0.]], device='cuda:1')\n", + "1 / 3 tensor([[0., 0.]], device='cuda:0')\n", + "2 / 3 tensor([[0., 0.]], device='cuda:1')\n", + "2 / 3 tensor([[0., 0.]], device='cuda:0')\n", + "3 / 3 tensor([[0., 0.]], device='cuda:1')\n", + "3 / 3 tensor([[0., 0.]], device='cuda:0')\n", + "evaluation metric - class 1: 0.0\n", + "evaluation metric - class 2: 0.0\n", + "avg_metric 0.0\n", "saved new best metric model\n", - "current epoch: 1 current mean dice: 0.0128 best mean dice: 0.0128 at epoch 1\n", + "current epoch: 1 current mean dice: 0.0000 best mean dice: 0.0000 at epoch 1\n", "----------\n", "epoch 2/2\n", - "learning rate is set to 6.25e-05\n", - "[2022-09-28 17:11:34] 1/16, train_loss: 0.7544\n", - "[2022-09-28 17:11:36] 2/16, train_loss: 0.8144\n", - "[2022-09-28 17:11:36] 3/16, train_loss: 0.8003\n", - "[2022-09-28 17:11:36] 4/16, train_loss: 0.7940\n", - "[2022-09-28 17:11:36] 5/16, train_loss: 0.7637\n", - "[2022-09-28 17:11:36] 6/16, train_loss: 0.8216\n", - "[2022-09-28 17:11:47] 7/16, train_loss: 0.7264\n", - "[2022-09-28 17:11:47] 8/16, train_loss: 0.8045\n", - "[2022-09-28 17:11:47] 9/16, train_loss: 0.7602\n", - "[2022-09-28 17:11:47] 10/16, train_loss: 0.8182\n", - "[2022-09-28 17:11:47] 11/16, train_loss: 0.8250\n", - "[2022-09-28 17:11:48] 12/16, train_loss: 0.7793\n", - "[2022-09-28 17:11:48] 13/16, train_loss: 0.7558\n", - "[2022-09-28 17:11:48] 14/16, train_loss: 0.7282\n", - "[2022-09-28 17:11:58] 15/16, train_loss: 0.7240\n", - "[2022-09-28 17:11:58] 16/16, train_loss: 0.8222\n", - "epoch 2 average loss: 0.7808, best mean dice: 0.0128 at epoch 1\n", - "1 / 8 tensor([[0.0241]], device='cuda:0')\n", - "2 / 8 tensor([[0.0193]], device='cuda:0')\n", - "3 / 8 tensor([[0.0025]], device='cuda:0')\n", - "4 / 8 tensor([[0.0060]], device='cuda:0')\n", - "5 / 8 tensor([[0.0138]], device='cuda:0')\n", - "6 / 8 tensor([[0.0090]], device='cuda:0')\n", - "7 / 8 tensor([[0.0102]], device='cuda:0')\n", - "8 / 8 tensor([[0.0121]], device='cuda:0')\n", - "evaluation metric - class 1: 0.01212325133383274\n", - "avg_metric 0.01212325133383274\n", - "current epoch: 2 current mean dice: 0.0121 best mean dice: 0.0128 at epoch 1\n", - "train completed, best_metric: 0.0128 at epoch: 1\n", - "0.012783133424818516\n", - "\", stderr=b'')\n" + "learning rate is set to 0.0125\n", + "[2022-10-18 08:43:16] 1/6, train_loss: 0.6629\n", + "[2022-10-18 08:43:16] 2/6, train_loss: 0.7294\n", + "[2022-10-18 08:43:16] 3/6, train_loss: 0.6376\n", + "[2022-10-18 08:43:16] 4/6, train_loss: 0.6884\n", + "[2022-10-18 08:43:16] 5/6, train_loss: 0.6723\n", + "[2022-10-18 08:43:16] 6/6, train_loss: 0.6182\n", + "epoch 2 average loss: 0.6574, best mean dice: 0.0000 at epoch 1\n", + "1 / 3 tensor([[0., 0.]], device='cuda:1')\n", + "1 / 3 tensor([[0., 0.]], device='cuda:0')\n", + "2 / 3 tensor([[0., 0.]], device='cuda:1')\n", + "2 / 3 tensor([[0., 0.]], device='cuda:0')\n", + "3 / 3 tensor([[0., 0.]], device='cuda:1')\n", + "3 / 3 tensor([[0., 0.]], device='cuda:0')\n", + "evaluation metric - class 1: 0.0\n", + "evaluation metric - class 2: 0.0\n", + "avg_metric 0.0\n", + "current epoch: 2 current mean dice: 0.0000 best mean dice: 0.0000 at epoch 1\n", + "train completed, best_metric: 0.0000 at epoch: 1\n", + "0.0\n", + "-1\n", + "\", stderr=b'WARNING:torch.distributed.run:\n", + "*****************************************\n", + "Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. \n", + "*****************************************\n", + "is_namedtuple is deprecated, please use the python checks instead\n", + "is_namedtuple is deprecated, please use the python checks instead\n", + "')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "\u001b[32m[I 2022-09-28 17:12:50,109]\u001b[0m Trial 0 finished with value: 0.012783133424818516 and parameters: {'learning_rate': 0.001}. Best is trial 0 with value: 0.012783133424818516.\u001b[0m\n" + "\u001b[32m[I 2022-10-18 08:43:19,053]\u001b[0m Trial 0 finished with value: 0.0 and parameters: {'learning_rate': 0.1}. Best is trial 0 with value: 0.0.\u001b[0m\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2022-09-28 17:12:50,835 - INFO - ./optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.1\n", - "2022-09-28 17:12:50,854 - INFO - Launching: python ./optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.1/scripts/train.py run --config_file='./optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.1/configs/network.yaml','./optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.1/configs/transforms_validate.yaml','./optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.1/configs/transforms_infer.yaml','./optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.1/configs/hyper_parameters.yaml','./optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.1/configs/transforms_train.yaml' --learning_rate=0.1\n", - "2022-09-28 17:15:58,897 - INFO - CompletedProcess(args=['python', './optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.1/scripts/train.py', 'run', \"--config_file='./optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.1/configs/network.yaml','./optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.1/configs/transforms_validate.yaml','./optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.1/configs/transforms_infer.yaml','./optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.1/configs/hyper_parameters.yaml','./optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.1/configs/transforms_train.yaml'\", '--learning_rate=0.1'], returncode=0, stdout=b\"[info] number of GPUs: 1\n", - "[info] world_size: 1\n", - "train_files: 32\n", - "val_files: 8\n", + "2022-10-18 08:43:19,079 - INFO - ./optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.001\n", + "2022-10-18 08:43:19,079 - INFO - Launching: torchrun --nnodes=1 --nproc_per_node=2 ./optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.001/scripts/train.py run --config_file='optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.001/configs/transforms_validate.yaml','optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.001/configs/transforms_train.yaml','optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.001/configs/transforms_infer.yaml','optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.001/configs/network.yaml','optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.001/configs/hyper_parameters.yaml' --learning_rate=0.001\n", + "2022-10-18 08:43:29,806 - INFO - CompletedProcess(args=['torchrun', '--nnodes=1', '--nproc_per_node=2', './optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.001/scripts/train.py', 'run', \"--config_file='optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.001/configs/transforms_validate.yaml','optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.001/configs/transforms_train.yaml','optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.001/configs/transforms_infer.yaml','optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.001/configs/network.yaml','optuna_learningrate_grid/segresnet2d_0_override_learning_rate_0.001/configs/hyper_parameters.yaml'\", '--learning_rate=0.001'], returncode=0, stdout=b\"[info] number of GPUs: 2\n", + "[info] number of GPUs: 2\n", + "2022-10-18 08:43:22,501 - Added key: store_based_barrier_key:1 to store for rank: 1\n", + "2022-10-18 08:43:22,501 - Added key: store_based_barrier_key:1 to store for rank: 0\n", + "2022-10-18 08:43:22,501 - Rank 1: Completed store-based barrier for key:store_based_barrier_key:1 with 2 nodes.\n", + "2022-10-18 08:43:22,501 - Rank 0: Completed store-based barrier for key:store_based_barrier_key:1 with 2 nodes.\n", + "[info] world_size: 2\n", + "[info] world_size: 2\n", + "train_files:train_files: 1212\n", + "\n", + "val_files: val_files:3\n", + " 3\n", "num_epochs 2\n", "num_epochs_per_validation 1\n", "[info] training from scratch\n", "[info] amp enabled\n", + "[info] training from scratch\n", "----------\n", "epoch 1/2\n", - "learning rate is set to 0.1\n", - "[2022-09-28 17:13:16] 1/16, train_loss: 0.8703\n", - "[2022-09-28 17:13:18] 2/16, train_loss: 0.5976\n", - "[2022-09-28 17:13:18] 3/16, train_loss: 0.5378\n", - "[2022-09-28 17:13:19] 4/16, train_loss: 0.5376\n", - "[2022-09-28 17:13:24] 5/16, train_loss: 0.4907\n", - "[2022-09-28 17:13:24] 6/16, train_loss: 0.4620\n", - "[2022-09-28 17:13:24] 7/16, train_loss: 0.4707\n", - "[2022-09-28 17:13:24] 8/16, train_loss: 0.4943\n", - "[2022-09-28 17:13:24] 9/16, train_loss: 0.5291\n", - "[2022-09-28 17:13:24] 10/16, train_loss: 0.4435\n", - "[2022-09-28 17:13:24] 11/16, train_loss: 0.5396\n", - "[2022-09-28 17:13:35] 12/16, train_loss: 0.4222\n", - "[2022-09-28 17:13:45] 13/16, train_loss: 0.4728\n", - "[2022-09-28 17:13:45] 14/16, train_loss: 0.4565\n", - "[2022-09-28 17:13:45] 15/16, train_loss: 0.4522\n", - "[2022-09-28 17:13:45] 16/16, train_loss: 0.4394\n", - "epoch 1 average loss: 0.5135, best mean dice: -1.0000 at epoch -1\n", - "1 / 8 tensor([[0.]], device='cuda:0')\n", - "2 / 8 tensor([[0.]], device='cuda:0')\n", - "3 / 8 tensor([[0.]], device='cuda:0')\n", - "4 / 8 tensor([[0.]], device='cuda:0')\n", - "5 / 8 tensor([[0.]], device='cuda:0')\n", - "6 / 8 tensor([[0.]], device='cuda:0')\n", - "7 / 8 tensor([[0.]], device='cuda:0')\n", - "8 / 8 tensor([[0.]], device='cuda:0')\n", - "evaluation metric - class 1: 0.0\n", - "avg_metric 0.0\n", + "learning rate is set to 0.001\n", + "[2022-10-18 08:43:24] 1/6, train_loss: 0.8798\n", + "2022-10-18 08:43:24,374 - Reducer buckets have been rebuilt in this iteration.\n", + "2022-10-18 08:43:24,458 - Reducer buckets have been rebuilt in this iteration.\n", + "[2022-10-18 08:43:24] 2/6, train_loss: 0.9273\n", + "[2022-10-18 08:43:24] 3/6, train_loss: 0.8508\n", + "[2022-10-18 08:43:24] 4/6, train_loss: 0.7524\n", + "[2022-10-18 08:43:24] 5/6, train_loss: 1.0487\n", + "[2022-10-18 08:43:24] 6/6, train_loss: 0.8540\n", + "epoch 1 average loss: 0.8866, best mean dice: -1.0000 at epoch -1\n", + "1 / 3 tensor([[0.0344, 0.0453]], device='cuda:1')\n", + "1 / 3 tensor([[0.0226, 0.0548]], device='cuda:0')\n", + "2 / 3 tensor([[0.0288, 0.0440]], device='cuda:1')\n", + "2 / 3 tensor([[0.0297, 0.0363]], device='cuda:0')\n", + "3 / 3 tensor([[0.0335, 0.0560]], device='cuda:1')\n", + "3 / 3 tensor([[0.0175, 0.0511]], device='cuda:0')\n", + "evaluation metric - class 1: 0.027750035127003986\n", + "evaluation metric - class 2: 0.0479105810324351\n", + "avg_metric 0.03783030807971954\n", "saved new best metric model\n", - "current epoch: 1 current mean dice: 0.0000 best mean dice: 0.0000 at epoch 1\n", + "current epoch: 1 current mean dice: 0.0378 best mean dice: 0.0378 at epoch 1\n", "----------\n", "epoch 2/2\n", - "learning rate is set to 0.00625\n", - "[2022-09-28 17:14:45] 1/16, train_loss: 0.4742\n", - "[2022-09-28 17:14:46] 2/16, train_loss: 0.4236\n", - "[2022-09-28 17:14:54] 3/16, train_loss: 0.4167\n", - "[2022-09-28 17:14:54] 4/16, train_loss: 0.4362\n", - "[2022-09-28 17:14:54] 5/16, train_loss: 0.4146\n", - "[2022-09-28 17:14:54] 6/16, train_loss: 0.4438\n", - "[2022-09-28 17:14:54] 7/16, train_loss: 0.4268\n", - "[2022-09-28 17:14:54] 8/16, train_loss: 0.4431\n", - "[2022-09-28 17:14:54] 9/16, train_loss: 0.4792\n", - "[2022-09-28 17:15:05] 10/16, train_loss: 0.4439\n", - "[2022-09-28 17:15:07] 11/16, train_loss: 0.4807\n", - "[2022-09-28 17:15:07] 12/16, train_loss: 0.5427\n", - "[2022-09-28 17:15:07] 13/16, train_loss: 0.4717\n", - "[2022-09-28 17:15:07] 14/16, train_loss: 0.4180\n", - "[2022-09-28 17:15:07] 15/16, train_loss: 0.4512\n", - "[2022-09-28 17:15:07] 16/16, train_loss: 0.4235\n", - "epoch 2 average loss: 0.4494, best mean dice: 0.0000 at epoch 1\n", - "1 / 8 tensor([[0.]], device='cuda:0')\n", - "2 / 8 tensor([[0.]], device='cuda:0')\n", - "3 / 8 tensor([[0.]], device='cuda:0')\n", - "4 / 8 tensor([[0.]], device='cuda:0')\n", - "5 / 8 tensor([[0.]], device='cuda:0')\n", - "6 / 8 tensor([[0.]], device='cuda:0')\n", - "7 / 8 tensor([[0.]], device='cuda:0')\n", - "8 / 8 tensor([[0.]], device='cuda:0')\n", - "evaluation metric - class 1: 0.0\n", - "avg_metric 0.0\n", - "current epoch: 2 current mean dice: 0.0000 best mean dice: 0.0000 at epoch 1\n", - "train completed, best_metric: 0.0000 at epoch: 1\n", - "0.0\n", - "\", stderr=b'')\n" + "learning rate is set to 0.000125\n", + "[2022-10-18 08:43:26] 1/6, train_loss: 0.9405\n", + "[2022-10-18 08:43:26] 2/6, train_loss: 0.8476\n", + "[2022-10-18 08:43:26] 3/6, train_loss: 0.8416\n", + "[2022-10-18 08:43:26] 4/6, train_loss: 0.8531\n", + "[2022-10-18 08:43:26] 5/6, train_loss: 0.8536\n", + "[2022-10-18 08:43:26] 6/6, train_loss: 0.8033\n", + "epoch 2 average loss: 0.8692, best mean dice: 0.0378 at epoch 1\n", + "1 / 3 tensor([[0.0355, 0.0456]], device='cuda:1')\n", + "1 / 3 tensor([[0.0217, 0.0551]], device='cuda:0')\n", + "2 / 3 tensor([[0.0291, 0.0444]], device='cuda:1')\n", + "2 / 3 tensor([[0.0299, 0.0364]], device='cuda:0')\n", + "3 / 3 tensor([[0.0331, 0.0563]], device='cuda:1')\n", + "3 / 3 tensor([[0.0174, 0.0515]], device='cuda:0')\n", + "evaluation metric - class 1: 0.027793101966381073\n", + "evaluation metric - class 2: 0.048210546374320984\n", + "avg_metric 0.03800182417035103\n", + "saved new best metric model\n", + "current epoch: 2 current mean dice: 0.0380 best mean dice: 0.0380 at epoch 2\n", + "train completed, best_metric: 0.0380 at epoch: 2\n", + "0.03800182417035103\n", + "-1\n", + "\", stderr=b'WARNING:torch.distributed.run:\n", + "*****************************************\n", + "Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. \n", + "*****************************************\n", + "is_namedtuple is deprecated, please use the python checks instead\n", + "is_namedtuple is deprecated, please use the python checks instead\n", + "')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "\u001b[32m[I 2022-09-28 17:15:59,029]\u001b[0m Trial 1 finished with value: 0.0 and parameters: {'learning_rate': 0.1}. Best is trial 0 with value: 0.012783133424818516.\u001b[0m\n" + "\u001b[32m[I 2022-10-18 08:43:29,812]\u001b[0m Trial 1 finished with value: 0.03800182417035103 and parameters: {'learning_rate': 0.001}. Best is trial 1 with value: 0.03800182417035103.\u001b[0m\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Best value: 0.012783133424818516 (params: {'learning_rate': 0.001})\n", + "Best value: 0.03800182417035103 (params: {'learning_rate': 0.001})\n", "\n" ] } diff --git a/auto3dseg/tasks/msd/Task04_Hippocampus/README.md b/auto3dseg/tasks/msd/Task04_Hippocampus/README.md new file mode 100644 index 0000000000..ac12811e72 --- /dev/null +++ b/auto3dseg/tasks/msd/Task04_Hippocampus/README.md @@ -0,0 +1,3 @@ +# MSD Dataset Task04 Hippocampus + +This repository provides a benmarking guide and recipe to train the template algorithms, validation performance, and is tested and maintained by NVIDIA. diff --git a/auto3dseg/tasks/msd/Task04_Hippocampus/input.yaml b/auto3dseg/tasks/msd/Task04_Hippocampus/input.yaml new file mode 100644 index 0000000000..af1bd4079a --- /dev/null +++ b/auto3dseg/tasks/msd/Task04_Hippocampus/input.yaml @@ -0,0 +1,10 @@ +name: Task04_Hippocampus +task: segmentation + +modality: MRI +datalist: ./datafolds/msd_task04_hippocampus_folds.json # list of files +dataroot: /data/Task04_Hippocampus # data location + +multigpu: True + +class_names: [val_acc_ant, val_acc_post] diff --git a/auto3dseg/tasks/msd/Task04_Hippocampus/msd_task04_hippocampus_folds.json b/auto3dseg/tasks/msd/Task04_Hippocampus/msd_task04_hippocampus_folds.json new file mode 100644 index 0000000000..102982e109 --- /dev/null +++ b/auto3dseg/tasks/msd/Task04_Hippocampus/msd_task04_hippocampus_folds.json @@ -0,0 +1,204 @@ +{ + "testing": [ + { + "image": "imagesTs/hippocampus_267.nii.gz" + }, + { + "image": "imagesTs/hippocampus_379.nii.gz" + }, + { + "image": "imagesTs/hippocampus_208.nii.gz" + }, + { + "image": "imagesTs/hippocampus_275.nii.gz" + }, + { + "image": "imagesTs/hippocampus_131.nii.gz" + }, + { + "image": "imagesTs/hippocampus_140.nii.gz" + }, + { + "image": "imagesTs/hippocampus_076.nii.gz" + }, + { + "image": "imagesTs/hippocampus_115.nii.gz" + }, + { + "image": "imagesTs/hippocampus_168.nii.gz" + }, + { + "image": "imagesTs/hippocampus_119.nii.gz" + }, + { + "image": "imagesTs/hippocampus_021.nii.gz" + }, + { + "image": "imagesTs/hippocampus_121.nii.gz" + }, + { + "image": "imagesTs/hippocampus_388.nii.gz" + }, + { + "image": "imagesTs/hippocampus_384.nii.gz" + }, + { + "image": "imagesTs/hippocampus_284.nii.gz" + }, + { + "image": "imagesTs/hippocampus_005.nii.gz" + } + ], + "training": [ + { + "fold": 0, + "image": "imagesTr/hippocampus_367.nii.gz", + "label": "labelsTr/hippocampus_367.nii.gz" + }, + { + "fold": 0, + "image": "imagesTr/hippocampus_304.nii.gz", + "label": "labelsTr/hippocampus_304.nii.gz" + }, + { + "fold": 0, + "image": "imagesTr/hippocampus_204.nii.gz", + "label": "labelsTr/hippocampus_204.nii.gz" + }, + { + "fold": 0, + "image": "imagesTr/hippocampus_279.nii.gz", + "label": "labelsTr/hippocampus_279.nii.gz" + }, + { + "fold": 0, + "image": "imagesTr/hippocampus_308.nii.gz", + "label": "labelsTr/hippocampus_308.nii.gz" + }, + { + "fold": 0, + "image": "imagesTr/hippocampus_375.nii.gz", + "label": "labelsTr/hippocampus_375.nii.gz" + }, + { + "fold": 1, + "image": "imagesTr/hippocampus_074.nii.gz", + "label": "labelsTr/hippocampus_074.nii.gz" + }, + { + "fold": 1, + "image": "imagesTr/hippocampus_174.nii.gz", + "label": "labelsTr/hippocampus_174.nii.gz" + }, + { + "fold": 1, + "image": "imagesTr/hippocampus_314.nii.gz", + "label": "labelsTr/hippocampus_314.nii.gz" + }, + { + "fold": 1, + "image": "imagesTr/hippocampus_269.nii.gz", + "label": "labelsTr/hippocampus_269.nii.gz" + }, + { + "fold": 1, + "image": "imagesTr/hippocampus_277.nii.gz", + "label": "labelsTr/hippocampus_277.nii.gz" + }, + { + "fold": 1, + "image": "imagesTr/hippocampus_318.nii.gz", + "label": "labelsTr/hippocampus_318.nii.gz" + }, + { + "fold": 2, + "image": "imagesTr/hippocampus_310.nii.gz", + "label": "labelsTr/hippocampus_310.nii.gz" + }, + { + "fold": 2, + "image": "imagesTr/hippocampus_373.nii.gz", + "label": "labelsTr/hippocampus_373.nii.gz" + }, + { + "fold": 2, + "image": "imagesTr/hippocampus_355.nii.gz", + "label": "labelsTr/hippocampus_355.nii.gz" + }, + { + "fold": 2, + "image": "imagesTr/hippocampus_328.nii.gz", + "label": "labelsTr/hippocampus_328.nii.gz" + }, + { + "fold": 2, + "image": "imagesTr/hippocampus_228.nii.gz", + "label": "labelsTr/hippocampus_228.nii.gz" + }, + { + "fold": 2, + "image": "imagesTr/hippocampus_181.nii.gz", + "label": "labelsTr/hippocampus_181.nii.gz" + }, + { + "fold": 3, + "image": "imagesTr/hippocampus_221.nii.gz", + "label": "labelsTr/hippocampus_221.nii.gz" + }, + { + "fold": 3, + "image": "imagesTr/hippocampus_096.nii.gz", + "label": "labelsTr/hippocampus_096.nii.gz" + }, + { + "fold": 3, + "image": "imagesTr/hippocampus_242.nii.gz", + "label": "labelsTr/hippocampus_242.nii.gz" + }, + { + "fold": 3, + "image": "imagesTr/hippocampus_188.nii.gz", + "label": "labelsTr/hippocampus_188.nii.gz" + }, + { + "fold": 3, + "image": "imagesTr/hippocampus_088.nii.gz", + "label": "labelsTr/hippocampus_088.nii.gz" + }, + { + "fold": 3, + "image": "imagesTr/hippocampus_084.nii.gz", + "label": "labelsTr/hippocampus_084.nii.gz" + }, + { + "fold": 4, + "image": "imagesTr/hippocampus_132.nii.gz", + "label": "labelsTr/hippocampus_132.nii.gz" + }, + { + "fold": 4, + "image": "imagesTr/hippocampus_385.nii.gz", + "label": "labelsTr/hippocampus_385.nii.gz" + }, + { + "fold": 4, + "image": "imagesTr/hippocampus_260.nii.gz", + "label": "labelsTr/hippocampus_260.nii.gz" + }, + { + "fold": 4, + "image": "imagesTr/hippocampus_360.nii.gz", + "label": "labelsTr/hippocampus_360.nii.gz" + }, + { + "fold": 4, + "image": "imagesTr/hippocampus_203.nii.gz", + "label": "labelsTr/hippocampus_203.nii.gz" + }, + { + "fold": 4, + "image": "imagesTr/hippocampus_303.nii.gz", + "label": "labelsTr/hippocampus_303.nii.gz" + } + ] +} From e32df20773a8c129c17862e0c810672bf82e9be7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 18 Oct 2022 09:12:42 +0000 Subject: [PATCH 2/5] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- auto3dseg/tasks/msd/Task04_Hippocampus/input.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auto3dseg/tasks/msd/Task04_Hippocampus/input.yaml b/auto3dseg/tasks/msd/Task04_Hippocampus/input.yaml index af1bd4079a..786a43ed42 100644 --- a/auto3dseg/tasks/msd/Task04_Hippocampus/input.yaml +++ b/auto3dseg/tasks/msd/Task04_Hippocampus/input.yaml @@ -1,7 +1,7 @@ name: Task04_Hippocampus task: segmentation -modality: MRI +modality: MRI datalist: ./datafolds/msd_task04_hippocampus_folds.json # list of files dataroot: /data/Task04_Hippocampus # data location From 7e3d84f135590b2551998bd0c1dedf4053abbfe2 Mon Sep 17 00:00:00 2001 From: Mingxin Zheng <18563433+mingxin-zheng@users.noreply.github.com> Date: Tue, 18 Oct 2022 17:14:18 +0800 Subject: [PATCH 3/5] undo the workaround Signed-off-by: Mingxin Zheng <18563433+mingxin-zheng@users.noreply.github.com> --- runner.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/runner.sh b/runner.sh index 1850689f0c..303d90d11a 100755 --- a/runner.sh +++ b/runner.sh @@ -82,7 +82,6 @@ pattern="-and -name '*' -and ! -wholename '*federated_learning*'\ -and ! -wholename '*video_seg*'\ -and ! -wholename '*tcia_dataset*'\ -and ! -wholename '*MRI_reconstruction*'\ - -and ! -wholename '*auto3dseg_autorunner_ref_api*'\ -and ! -wholename '*get_started*'" kernelspec="python3" From 91aeeb013cc0a2515284917349b6f08d7677475c Mon Sep 17 00:00:00 2001 From: Mingxin Zheng <18563433+mingxin-zheng@users.noreply.github.com> Date: Tue, 18 Oct 2022 17:16:36 +0800 Subject: [PATCH 4/5] change epochs from 2000 to 2 Signed-off-by: Mingxin Zheng <18563433+mingxin-zheng@users.noreply.github.com> --- auto3dseg/notebooks/auto3dseg_autorunner_ref_api.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auto3dseg/notebooks/auto3dseg_autorunner_ref_api.ipynb b/auto3dseg/notebooks/auto3dseg_autorunner_ref_api.ipynb index ad9ebfe1a2..a66f5ba0bc 100644 --- a/auto3dseg/notebooks/auto3dseg_autorunner_ref_api.ipynb +++ b/auto3dseg/notebooks/auto3dseg_autorunner_ref_api.ipynb @@ -305,7 +305,7 @@ "source": [ "# The training params are optional. If you don't want to change the default settings, you can use either `train()` or `train({})` in 2.3.2 \n", "\n", - "max_epochs = 2000\n", + "max_epochs = 2 # change epoch number to 2 to cut down the notebook running time\n", "\n", "# safeguard to ensure max_epochs is greater or equal to 2\n", "max_epochs = max(max_epochs, 2)\n", From 21822250f0c5a1432f9c4178b6eafe34084fe095 Mon Sep 17 00:00:00 2001 From: Wenqi Li <831580+wyli@users.noreply.github.com> Date: Tue, 18 Oct 2022 15:39:53 +0100 Subject: [PATCH 5/5] Update README.md --- auto3dseg/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auto3dseg/README.md b/auto3dseg/README.md index 83032c7da7..4bab6b4511 100644 --- a/auto3dseg/README.md +++ b/auto3dseg/README.md @@ -43,7 +43,7 @@ We have demonstrated preliminary results of public datasets are described in the - [INSTANCE22](tasks/instance22) - [Beyond the Cranial Vault (BTCV) Abdomen Dataset](tasks/btcv) - Medical Segmentation Decathlon (MSD) Dataset - - [Task04 Task04_Hippocampus](tasks/msd/Task04_Hippocampus) + - [Task04 Task04_Hippocampus](tasks/msd/Task04_Hippocampus) - [Task05 Prostate](tasks/msd/Task05_Prostate) - [Task09 Spleen](tasks/msd/Task09_Spleen)