From f16174a3e81b1a27ac2378757f814c362e5329e4 Mon Sep 17 00:00:00 2001 From: Hao Chen Date: Wed, 19 Jul 2023 06:21:35 -0400 Subject: [PATCH 01/12] [Update] resolve requirements.txt conflicts --- requirements.txt | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index 296588f22..6a1f004b3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,9 +16,6 @@ joblib==1.2.0 kiwisolver==1.4.3 Markdown==3.3.7 matplotlib==3.5.2 -mkl-fft>=1.3.0 -mkl-random==1.2.2 -mkl-service==2.4.0 networkx==2.6.3 numpy oauthlib==3.2.1 @@ -51,10 +48,10 @@ tifffile==2021.11.2 timm==0.5.4 tokenizers==0.12.1 torch>=1.12.0 -torchaudio==0.12.0 +torchaudio>=0.12.0 torchvision>=0.13.0 tqdm==4.64.0 -transformers==4.20.1 +transformers>=4.20.1 typing-extensions==4.3.0 urllib3==1.26.9 Werkzeug==2.1.2 From 37d7561dd5edbdbd8041f70edcc6df78024dde4c Mon Sep 17 00:00:00 2001 From: Hhhhhhao Date: Wed, 19 Jul 2023 06:34:10 -0400 Subject: [PATCH 02/12] [Fix] Fix mean teacher bug in #102 --- semilearn/algorithms/meanteacher/meanteacher.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/semilearn/algorithms/meanteacher/meanteacher.py b/semilearn/algorithms/meanteacher/meanteacher.py index f776a8434..9fd515307 100644 --- a/semilearn/algorithms/meanteacher/meanteacher.py +++ b/semilearn/algorithms/meanteacher/meanteacher.py @@ -52,8 +52,8 @@ def train_step(self, x_lb, y_lb, x_ulb_w, x_ulb_s): self.bn_controller.freeze_bn(self.model) outs_x_ulb_s = self.model(x_ulb_s) - logits_x_ulb_s = outs_x_ulb_w['logits'] - feats_x_ulb_s = outs_x_ulb_w['feat'] + logits_x_ulb_s = outs_x_ulb_s['logits'] + feats_x_ulb_s = outs_x_ulb_s['feat'] self.bn_controller.unfreeze_bn(self.model) feat_dict = {'x_lb':feats_x_lb, 'x_ulb_w':feats_x_ulb_w, 'x_ulb_s':feats_x_ulb_s} From 2bfe15a035c725d470d176fdf62b43654d2db787 Mon Sep 17 00:00:00 2001 From: Hhhhhhao Date: Wed, 19 Jul 2023 21:56:01 -0400 Subject: [PATCH 03/12] [Fix] Fix DebiasPL bug --- semilearn/imb_algorithms/debiaspl/debiaspl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/semilearn/imb_algorithms/debiaspl/debiaspl.py b/semilearn/imb_algorithms/debiaspl/debiaspl.py index 488afc1de..429ac2af8 100644 --- a/semilearn/imb_algorithms/debiaspl/debiaspl.py +++ b/semilearn/imb_algorithms/debiaspl/debiaspl.py @@ -49,7 +49,7 @@ def set_hooks(self): def compute_prob(self, logits): # update p_hat probs = super().compute_prob(logits) - delta_p = probs.mean() + delta_p = probs.mean(dim=0) self.p_hat = self.ema_m * self.p_hat + (1 - self.ema_p) * delta_p return super().compute_prob(logits - self.tau * torch.log(self.p_hat)) From fce945b24379efa6fe73ce3f2f2a41c5e2d7a2dd Mon Sep 17 00:00:00 2001 From: Hhhhhhao Date: Wed, 19 Jul 2023 22:03:39 -0400 Subject: [PATCH 04/12] [Fix] Fix potential sample data bug in #119 --- semilearn/datasets/utils.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/semilearn/datasets/utils.py b/semilearn/datasets/utils.py index 42bc9404b..aa7809512 100644 --- a/semilearn/datasets/utils.py +++ b/semilearn/datasets/utils.py @@ -52,10 +52,6 @@ def split_ssl_data(args, data, targets, num_classes, return data[lb_idx], targets[lb_idx], data[ulb_idx], targets[ulb_idx] -def sample_labeled_data(): - pass - - def sample_labeled_unlabeled_data(args, data, target, num_classes, lb_num_labels, ulb_num_labels=None, lb_imbalance_ratio=1.0, ulb_imbalance_ratio=1.0, @@ -88,7 +84,7 @@ def sample_labeled_unlabeled_data(args, data, target, num_classes, if ulb_imbalance_ratio == 1.0: # balanced setting if ulb_num_labels is None or ulb_num_labels == 'None': - pass # ulb_samples_per_class = [int(len(data) / num_classes) - lb_samples_per_class[c] for c in range(num_classes)] # [int(len(data) / num_classes) - int(lb_num_labels / num_classes)] * num_classes + ulb_samples_per_class = None else: assert ulb_num_labels % num_classes == 0, "ulb_num_labels must be dividable by num_classes in balanced setting" ulb_samples_per_class = [int(ulb_num_labels / num_classes)] * num_classes @@ -104,7 +100,7 @@ def sample_labeled_unlabeled_data(args, data, target, num_classes, idx = np.where(target == c)[0] np.random.shuffle(idx) lb_idx.extend(idx[:lb_samples_per_class[c]]) - if ulb_num_labels is None or ulb_num_labels == 'None': + if ulb_samples_per_class is None: ulb_idx.extend(idx[lb_samples_per_class[c]:]) else: ulb_idx.extend(idx[lb_samples_per_class[c]:lb_samples_per_class[c]+ulb_samples_per_class[c]]) From de1a33c9aba6e2734f253beee5d7f2b8e22457f7 Mon Sep 17 00:00:00 2001 From: Hhhhhhao Date: Wed, 19 Jul 2023 22:32:17 -0400 Subject: [PATCH 05/12] [Update] Add auto issue/pr closer --- .github/workflows/blank.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/blank.yml b/.github/workflows/blank.yml index 6e7374617..494ff3d63 100644 --- a/.github/workflows/blank.yml +++ b/.github/workflows/blank.yml @@ -34,3 +34,21 @@ jobs: run: | echo Add other actions to build, echo test, and deploy your project. + +name: 'Close stale issues and PR' +on: + schedule: + - cron: '30 1 * * *' + +jobs: + stale: + runs-on: ubuntu-latest + steps: + - uses: actions/stale@v8 + with: + stale-issue-message: 'This issue is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 5 days.' + stale-pr-message: 'This PR is stale because it has been open 45 days with no activity. Remove stale label or comment or this will be closed in 10 days.' + close-issue-message: 'This issue was closed because it has been stalled for 5 days with no activity.' + days-before-stale: 30 + days-before-close: 5 + days-before-pr-close: -1 \ No newline at end of file From 26efb671cd9dd5523928696de9ee9c025dfcdec0 Mon Sep 17 00:00:00 2001 From: Hhhhhhao Date: Wed, 19 Jul 2023 22:32:34 -0400 Subject: [PATCH 06/12] [Update] Update requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6a1f004b3..30730b243 100644 --- a/requirements.txt +++ b/requirements.txt @@ -51,7 +51,7 @@ torch>=1.12.0 torchaudio>=0.12.0 torchvision>=0.13.0 tqdm==4.64.0 -transformers>=4.20.1 +transformers==4.20.1 typing-extensions==4.3.0 urllib3==1.26.9 Werkzeug==2.1.2 From e3c444c222eeeadcd0c771ac8a548abdc1e374f9 Mon Sep 17 00:00:00 2001 From: Hhhhhhao Date: Wed, 19 Jul 2023 22:33:10 -0400 Subject: [PATCH 07/12] [Fix] Fix bug in #74 --- semilearn/lighting/trainer.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/semilearn/lighting/trainer.py b/semilearn/lighting/trainer.py index 8567c845e..afdf434de 100644 --- a/semilearn/lighting/trainer.py +++ b/semilearn/lighting/trainer.py @@ -48,19 +48,24 @@ def fit(self, train_lb_loader, train_ulb_loader, eval_loader): bar = Bar('Processing', max=len(train_lb_loader)) self.algorithm.model.train() + self.algorithm.call_hook("before_train_epoch") for data_lb, data_ulb in zip(train_lb_loader, train_ulb_loader): if self.algorithm.it > self.config.num_train_iter: break + self.algorithm.call_hook("before_train_step") result = self.algorithm.train_step(**self.algorithm.process_batch(**data_lb, **data_ulb)) - + self.algorithm.call_hook("after_train_step") + bar.suffix = ("Iter: {batch:4}/{iter:4}.".format(batch=self.algorithm.it, iter=len(train_lb_loader))) bar.next() self.algorithm.it += 1 bar.finish() + self.algorithm.call_hook("after_train_epoch") + # validate result = self.evaluate(eval_loader) From 9c9401c40507e730a1a74b4625e625df361d3342 Mon Sep 17 00:00:00 2001 From: Hhhhhhao Date: Wed, 19 Jul 2023 22:51:49 -0400 Subject: [PATCH 08/12] [Fix] Fix amp lighting bug in #123 --- semilearn/lighting/config.py | 2 +- setup.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/semilearn/lighting/config.py b/semilearn/lighting/config.py index 6a10c5a43..a52db2261 100644 --- a/semilearn/lighting/config.py +++ b/semilearn/lighting/config.py @@ -69,7 +69,7 @@ def get_config(config): ## core algorithm setting parser.add_argument('-alg', '--algorithm', type=str, default='fixmatch', help='ssl algorithm') parser.add_argument('--use_cat', type=str2bool, default=True, help='use cat operation in algorithms') - parser.add_argument('--use_amp', type=str2bool, default=False, help='use mixed precision training or not') + parser.add_argument('--amp', type=str2bool, default=False, help='use mixed precision training or not') parser.add_argument('--clip_grad', type=float, default=0) ## imbalance algorithm setting diff --git a/setup.py b/setup.py index 00906b535..aa7a8d7f6 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ setup( name='semilearn', - version='0.3.0', + version='0.3.1a', description='Unfied Semi-Supervised Learning Benchmark', long_description=long_description, long_description_content_type='text/markdown', @@ -41,5 +41,5 @@ include_package_data=True, # install_requires=['torch >= 1.8', 'torchvision', 'torchaudio', 'transformers', 'timm', 'progress', 'ruamel.yaml', 'scikit-image', 'scikit-learn', 'tensorflow', ''], install_requires=install_requires, - python_requires='>=3.8', + python_requires='>=3.9', ) From 59151a7b337136c112f5de69261b2d8b040e9165 Mon Sep 17 00:00:00 2001 From: Hhhhhhao Date: Wed, 19 Jul 2023 23:47:43 -0400 Subject: [PATCH 09/12] [Fix] Fix notebook bugs --- notebooks/Beginner_Example.ipynb | 388 ++++++++++++++++++++++++++----- notebooks/Custom_Algorithm.ipynb | 367 +++++++++++++++++++++++++---- notebooks/Custom_Dataset.ipynb | 251 +++----------------- semilearn/lighting/trainer.py | 15 +- 4 files changed, 689 insertions(+), 332 deletions(-) diff --git a/notebooks/Beginner_Example.ipynb b/notebooks/Beginner_Example.ipynb index e1fd15bd4..32b395375 100644 --- a/notebooks/Beginner_Example.ipynb +++ b/notebooks/Beginner_Example.ipynb @@ -33,8 +33,18 @@ "name": "#%%\n" } }, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/media/Zeus/haoc/miniconda/envs/test_semilearn_031/lib/python3.9/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], "source": [ + "import semilearn\n", "from semilearn import get_dataset, get_data_loader, get_net_builder, get_algorithm, get_config, Trainer\n" ] }, @@ -60,13 +70,6 @@ } }, "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "here 1\n" - ] - }, { "name": "stderr", "output_type": "stream", @@ -83,10 +86,10 @@ " 'pretrain_path': 'https://github.com/microsoft/Semi-supervised-learning/releases/download/v.0.0.0/vit_tiny_patch2_32_mlp_im_1k_32.pth',\n", "\n", " # optimization configs\n", - " 'epoch': 1, # set to 100\n", - " 'num_train_iter': 5000, # set to 102400\n", - " 'num_eval_iter': 500, # set to 1024\n", - " 'num_log_iter': 50, # set to 256\n", + " 'epoch': 1, \n", + " 'num_train_iter': 5000, \n", + " 'num_eval_iter': 500, \n", + " 'num_log_iter': 50, \n", " 'optim': 'AdamW',\n", " 'lr': 5e-4,\n", " 'layer_decay': 0.5,\n", @@ -225,41 +228,306 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch: 0\n" + "Epoch: 0\n", + "50 iteration USE_EMA: True, train/sup_loss: 1.9210, train/unsup_loss: 0.0000, train/total_loss: 1.9210, train/util_ratio: 0.0000, train/run_time: 0.1366, lr: 0.0005, train/prefecth_time: 0.0040 \n", + "100 iteration USE_EMA: True, train/sup_loss: 0.9313, train/unsup_loss: 0.0000, train/total_loss: 0.9313, train/util_ratio: 0.0000, train/run_time: 0.1369, lr: 0.0005, train/prefecth_time: 0.0037 \n", + "150 iteration USE_EMA: True, train/sup_loss: 0.4322, train/unsup_loss: 0.0000, train/total_loss: 0.4322, train/util_ratio: 0.0000, train/run_time: 0.1365, lr: 0.0005, train/prefecth_time: 0.0037 \n", + "200 iteration USE_EMA: True, train/sup_loss: 0.2618, train/unsup_loss: 0.0000, train/total_loss: 0.2618, train/util_ratio: 0.0000, train/run_time: 0.1383, lr: 0.0005, train/prefecth_time: 0.0040 \n", + "250 iteration USE_EMA: True, train/sup_loss: 0.4954, train/unsup_loss: 0.0902, train/total_loss: 0.5856, train/util_ratio: 0.0938, train/run_time: 0.1384, lr: 0.0005, train/prefecth_time: 0.0038 \n", + "300 iteration USE_EMA: True, train/sup_loss: 0.1285, train/unsup_loss: 0.0065, train/total_loss: 0.1350, train/util_ratio: 0.0938, train/run_time: 0.1387, lr: 0.0005, train/prefecth_time: 0.0037 \n", + "350 iteration USE_EMA: True, train/sup_loss: 0.0994, train/unsup_loss: 0.0367, train/total_loss: 0.1361, train/util_ratio: 0.0938, train/run_time: 0.1388, lr: 0.0005, train/prefecth_time: 0.0040 \n", + "400 iteration USE_EMA: True, train/sup_loss: 0.0280, train/unsup_loss: 0.0182, train/total_loss: 0.0462, train/util_ratio: 0.1250, train/run_time: 0.1393, lr: 0.0005, train/prefecth_time: 0.0035 \n", + "450 iteration USE_EMA: True, train/sup_loss: 0.0223, train/unsup_loss: 0.3827, train/total_loss: 0.4050, train/util_ratio: 0.4062, train/run_time: 0.1398, lr: 0.0005, train/prefecth_time: 0.0039 \n", + "validating...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/media/Zeus/haoc/miniconda/envs/test_semilearn_031/lib/python3.9/site-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "confusion matrix:\n", + "[[0. 0. 0. 0. 1. 0. 0. 0. 0. 0.]\n", + " [0. 0. 0. 0. 1. 0. 0. 0. 0. 0.]\n", + " [0. 0. 0. 0. 1. 0. 0. 0. 0. 0.]\n", + " [0. 0. 0. 0. 1. 0. 0. 0. 0. 0.]\n", + " [0. 0. 0. 0. 1. 0. 0. 0. 0. 0.]\n", + " [0. 0. 0. 0. 1. 0. 0. 0. 0. 0.]\n", + " [0. 0. 0. 0. 1. 0. 0. 0. 0. 0.]\n", + " [0. 0. 0. 0. 1. 0. 0. 0. 0. 0.]\n", + " [0. 0. 0. 0. 1. 0. 0. 0. 0. 0.]\n", + " [0. 0. 0. 0. 1. 0. 0. 0. 0. 0.]]\n", + "model saved: ./saved_models/fixmatch/latest_model.pth\n", + "model saved: ./saved_models/fixmatch/model_best.pth\n", + "500 iteration, USE_EMA: True, train/sup_loss: 0.0236, train/unsup_loss: 0.0640, train/total_loss: 0.0877, train/util_ratio: 0.2812, train/run_time: 0.1395, eval/loss: 5.5914, eval/top-1-acc: 0.1000, eval/balanced_acc: 0.1000, eval/precision: 0.0100, eval/recall: 0.1000, eval/F1: 0.0182, lr: 0.0005, train/prefecth_time: 0.0038 BEST_EVAL_ACC: 0.1000, at 500 iters\n", + "550 iteration USE_EMA: True, train/sup_loss: 0.0866, train/unsup_loss: 0.2069, train/total_loss: 0.2936, train/util_ratio: 0.2812, train/run_time: 0.1393, lr: 0.0005, train/prefecth_time: 0.0038 \n", + "600 iteration USE_EMA: True, train/sup_loss: 0.0649, train/unsup_loss: 0.0583, train/total_loss: 0.1232, train/util_ratio: 0.3750, train/run_time: 0.1393, lr: 0.0005, train/prefecth_time: 0.0035 \n", + "650 iteration USE_EMA: True, train/sup_loss: 0.0256, train/unsup_loss: 0.0846, train/total_loss: 0.1102, train/util_ratio: 0.5625, train/run_time: 0.1402, lr: 0.0005, train/prefecth_time: 0.0031 \n", + "700 iteration USE_EMA: True, train/sup_loss: 0.0195, train/unsup_loss: 0.2254, train/total_loss: 0.2449, train/util_ratio: 0.5938, train/run_time: 0.1409, lr: 0.0005, train/prefecth_time: 0.0038 \n", + "750 iteration USE_EMA: True, train/sup_loss: 0.0331, train/unsup_loss: 0.1504, train/total_loss: 0.1835, train/util_ratio: 0.6250, train/run_time: 0.1408, lr: 0.0005, train/prefecth_time: 0.0039 \n", + "800 iteration USE_EMA: True, train/sup_loss: 0.0006, train/unsup_loss: 0.0241, train/total_loss: 0.0247, train/util_ratio: 0.5938, train/run_time: 0.1414, lr: 0.0005, train/prefecth_time: 0.0037 \n", + "850 iteration USE_EMA: True, train/sup_loss: 0.0085, train/unsup_loss: 0.1780, train/total_loss: 0.1864, train/util_ratio: 0.5000, train/run_time: 0.1413, lr: 0.0005, train/prefecth_time: 0.0050 \n", + "900 iteration USE_EMA: True, train/sup_loss: 0.0268, train/unsup_loss: 0.5602, train/total_loss: 0.5870, train/util_ratio: 0.5312, train/run_time: 0.1402, lr: 0.0005, train/prefecth_time: 0.0037 \n", + "950 iteration USE_EMA: True, train/sup_loss: 0.0060, train/unsup_loss: 0.1938, train/total_loss: 0.1998, train/util_ratio: 0.5312, train/run_time: 0.1400, lr: 0.0005, train/prefecth_time: 0.0035 \n", + "validating...\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "/home/haoc/miniconda3/envs/test/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "[2022-12-18 14:01:08,644 INFO] confusion matrix\n", - "[2022-12-18 14:01:08,646 INFO] [[0. 0. 1. 0. 0. 0. 0. 0. 0. 0.]\n", - " [0. 0. 1. 0. 0. 0. 0. 0. 0. 0.]\n", - " [0. 0. 1. 0. 0. 0. 0. 0. 0. 0.]\n", - " [0. 0. 1. 0. 0. 0. 0. 0. 0. 0.]\n", - " [0. 0. 1. 0. 0. 0. 0. 0. 0. 0.]\n", - " [0. 0. 1. 0. 0. 0. 0. 0. 0. 0.]\n", - " [0. 0. 1. 0. 0. 0. 0. 0. 0. 0.]\n", - " [0. 0. 1. 0. 0. 0. 0. 0. 0. 0.]\n", - " [0. 0. 1. 0. 0. 0. 0. 0. 0. 0.]\n", - " [0. 0. 1. 0. 0. 0. 0. 0. 0. 0.]]\n", - "[2022-12-18 14:01:08,649 INFO] evaluation metric\n", - "[2022-12-18 14:01:08,650 INFO] acc: 0.1000\n", - "[2022-12-18 14:01:08,652 INFO] precision: 0.0100\n", - "[2022-12-18 14:01:08,652 INFO] recall: 0.1000\n", - "[2022-12-18 14:01:08,653 INFO] f1: 0.0182\n", - "[2022-12-18 14:01:09,174 INFO] Best acc 0.1000 at epoch 0\n", - "[2022-12-18 14:01:09,176 INFO] Training finished.\n" + "/media/Zeus/haoc/miniconda/envs/test_semilearn_031/lib/python3.9/site-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ + "confusion matrix:\n", + "[[0.127 0. 0. 0. 0.484 0. 0.367 0.007 0.004 0.011]\n", + " [0. 0.277 0. 0. 0.568 0. 0.049 0.039 0. 0.067]\n", + " [0. 0. 0. 0. 0.93 0. 0.066 0.004 0. 0. ]\n", + " [0. 0. 0. 0.002 0.92 0. 0.065 0.012 0. 0.001]\n", + " [0. 0. 0. 0. 0.995 0. 0.003 0.002 0. 0. ]\n", + " [0. 0. 0. 0. 0.955 0.002 0.034 0.009 0. 0. ]\n", + " [0. 0. 0. 0. 0.445 0. 0.555 0. 0. 0. ]\n", + " [0. 0. 0. 0. 0.847 0. 0.003 0.15 0. 0. ]\n", + " [0.002 0. 0. 0. 0.822 0. 0.142 0.009 0.014 0.011]\n", + " [0. 0. 0. 0. 0.446 0. 0.022 0.005 0. 0.527]]\n", + "model saved: ./saved_models/fixmatch/latest_model.pth\n", + "model saved: ./saved_models/fixmatch/model_best.pth\n", + "1000 iteration, USE_EMA: True, train/sup_loss: 0.0011, train/unsup_loss: 0.2126, train/total_loss: 0.2137, train/util_ratio: 0.7500, train/run_time: 0.1401, eval/loss: 2.2794, eval/top-1-acc: 0.2649, eval/balanced_acc: 0.2649, eval/precision: 0.6809, eval/recall: 0.2649, eval/F1: 0.2307, lr: 0.0005, train/prefecth_time: 0.0037 BEST_EVAL_ACC: 0.2649, at 1000 iters\n", + "1050 iteration USE_EMA: True, train/sup_loss: 0.0556, train/unsup_loss: 0.2425, train/total_loss: 0.2981, train/util_ratio: 0.6562, train/run_time: 0.1413, lr: 0.0005, train/prefecth_time: 0.0038 \n", + "1100 iteration USE_EMA: True, train/sup_loss: 0.0058, train/unsup_loss: 0.4130, train/total_loss: 0.4188, train/util_ratio: 0.7188, train/run_time: 0.1409, lr: 0.0005, train/prefecth_time: 0.0038 \n", + "1150 iteration USE_EMA: True, train/sup_loss: 0.0017, train/unsup_loss: 0.4102, train/total_loss: 0.4120, train/util_ratio: 0.6562, train/run_time: 0.1404, lr: 0.0005, train/prefecth_time: 0.0032 \n", + "1200 iteration USE_EMA: True, train/sup_loss: 0.0031, train/unsup_loss: 0.2593, train/total_loss: 0.2624, train/util_ratio: 0.6562, train/run_time: 0.1407, lr: 0.0005, train/prefecth_time: 0.0035 \n", + "1250 iteration USE_EMA: True, train/sup_loss: 0.0039, train/unsup_loss: 0.2743, train/total_loss: 0.2783, train/util_ratio: 0.7188, train/run_time: 0.1409, lr: 0.0005, train/prefecth_time: 0.0037 \n", + "1300 iteration USE_EMA: True, train/sup_loss: 0.0655, train/unsup_loss: 0.1830, train/total_loss: 0.2484, train/util_ratio: 0.6562, train/run_time: 0.1407, lr: 0.0005, train/prefecth_time: 0.0038 \n", + "1350 iteration USE_EMA: True, train/sup_loss: 0.0027, train/unsup_loss: 0.0781, train/total_loss: 0.0808, train/util_ratio: 0.5625, train/run_time: 0.1414, lr: 0.0005, train/prefecth_time: 0.0035 \n", + "1400 iteration USE_EMA: True, train/sup_loss: 0.0141, train/unsup_loss: 0.2426, train/total_loss: 0.2568, train/util_ratio: 0.7188, train/run_time: 0.1403, lr: 0.0005, train/prefecth_time: 0.0036 \n", + "1450 iteration USE_EMA: True, train/sup_loss: 0.0057, train/unsup_loss: 0.3173, train/total_loss: 0.3229, train/util_ratio: 0.8750, train/run_time: 0.1414, lr: 0.0005, train/prefecth_time: 0.0037 \n", + "validating...\n", + "confusion matrix:\n", + "[[0.922 0.001 0.001 0.001 0.005 0. 0.03 0.001 0.017 0.022]\n", + " [0.001 0.963 0. 0. 0.003 0. 0.001 0. 0.001 0.031]\n", + " [0.024 0. 0.436 0.002 0.419 0.005 0.1 0.014 0. 0. ]\n", + " [0.01 0.001 0.002 0.5 0.283 0.085 0.085 0.029 0.002 0.003]\n", + " [0. 0. 0. 0.001 0.981 0. 0.007 0.011 0. 0. ]\n", + " [0.001 0. 0. 0.015 0.214 0.692 0.024 0.054 0. 0. ]\n", + " [0.001 0. 0.001 0. 0.01 0.001 0.986 0. 0.001 0. ]\n", + " [0.005 0. 0. 0.006 0.374 0.008 0.002 0.605 0. 0. ]\n", + " [0.023 0.011 0. 0. 0.048 0. 0.025 0.002 0.865 0.026]\n", + " [0.003 0.012 0. 0. 0. 0. 0.002 0. 0.001 0.982]]\n", + "model saved: ./saved_models/fixmatch/latest_model.pth\n", + "model saved: ./saved_models/fixmatch/model_best.pth\n", + "1500 iteration, USE_EMA: True, train/sup_loss: 0.0110, train/unsup_loss: 0.1831, train/total_loss: 0.1942, train/util_ratio: 0.6250, train/run_time: 0.1408, eval/loss: 0.6494, eval/top-1-acc: 0.7932, eval/balanced_acc: 0.7932, eval/precision: 0.8668, eval/recall: 0.7932, eval/F1: 0.7963, lr: 0.0005, train/prefecth_time: 0.0036 BEST_EVAL_ACC: 0.7932, at 1500 iters\n", + "1550 iteration USE_EMA: True, train/sup_loss: 0.0835, train/unsup_loss: 0.3773, train/total_loss: 0.4608, train/util_ratio: 0.6875, train/run_time: 0.1409, lr: 0.0005, train/prefecth_time: 0.0039 \n", + "1600 iteration USE_EMA: True, train/sup_loss: 0.0115, train/unsup_loss: 0.5431, train/total_loss: 0.5546, train/util_ratio: 0.7812, train/run_time: 0.1409, lr: 0.0005, train/prefecth_time: 0.0040 \n", + "1650 iteration USE_EMA: True, train/sup_loss: 0.0255, train/unsup_loss: 0.1681, train/total_loss: 0.1937, train/util_ratio: 0.7188, train/run_time: 0.1412, lr: 0.0004, train/prefecth_time: 0.0039 \n", + "1700 iteration USE_EMA: True, train/sup_loss: 0.0012, train/unsup_loss: 0.1483, train/total_loss: 0.1495, train/util_ratio: 0.5625, train/run_time: 0.1412, lr: 0.0004, train/prefecth_time: 0.0039 \n", + "1750 iteration USE_EMA: True, train/sup_loss: 0.0016, train/unsup_loss: 0.0522, train/total_loss: 0.0538, train/util_ratio: 0.6250, train/run_time: 0.1405, lr: 0.0004, train/prefecth_time: 0.0041 \n", + "1800 iteration USE_EMA: True, train/sup_loss: 0.0082, train/unsup_loss: 0.2608, train/total_loss: 0.2690, train/util_ratio: 0.8125, train/run_time: 0.1415, lr: 0.0004, train/prefecth_time: 0.0037 \n", + "1850 iteration USE_EMA: True, train/sup_loss: 0.0011, train/unsup_loss: 0.1171, train/total_loss: 0.1182, train/util_ratio: 0.8438, train/run_time: 0.1412, lr: 0.0004, train/prefecth_time: 0.0042 \n", + "1900 iteration USE_EMA: True, train/sup_loss: 0.0348, train/unsup_loss: 0.1308, train/total_loss: 0.1656, train/util_ratio: 0.6562, train/run_time: 0.1411, lr: 0.0004, train/prefecth_time: 0.0037 \n", + "1950 iteration USE_EMA: True, train/sup_loss: 0.0104, train/unsup_loss: 0.0852, train/total_loss: 0.0956, train/util_ratio: 0.5625, train/run_time: 0.1412, lr: 0.0004, train/prefecth_time: 0.0040 \n", + "validating...\n", + "confusion matrix:\n", + "[[0.946 0.003 0.001 0.001 0.002 0. 0.007 0. 0.019 0.021]\n", + " [0. 0.979 0. 0. 0. 0. 0.001 0. 0.001 0.019]\n", + " [0.04 0. 0.701 0.006 0.177 0.012 0.048 0.016 0. 0. ]\n", + " [0.009 0.001 0.004 0.769 0.059 0.1 0.039 0.015 0.002 0.002]\n", + " [0. 0. 0.001 0.005 0.96 0.001 0.008 0.024 0.001 0. ]\n", + " [0.001 0. 0.001 0.024 0.015 0.915 0.007 0.037 0. 0. ]\n", + " [0.001 0. 0.003 0. 0.002 0.001 0.992 0. 0.001 0. ]\n", + " [0.005 0. 0.003 0.009 0.109 0.03 0.001 0.843 0. 0. ]\n", + " [0.03 0.007 0. 0. 0.005 0. 0.005 0.002 0.931 0.02 ]\n", + " [0.004 0.025 0. 0. 0. 0. 0. 0. 0.001 0.97 ]]\n", + "model saved: ./saved_models/fixmatch/latest_model.pth\n", + "model saved: ./saved_models/fixmatch/model_best.pth\n", + "2000 iteration, USE_EMA: True, train/sup_loss: 0.0046, train/unsup_loss: 0.1097, train/total_loss: 0.1143, train/util_ratio: 0.7500, train/run_time: 0.1419, eval/loss: 0.3166, eval/top-1-acc: 0.9006, eval/balanced_acc: 0.9006, eval/precision: 0.9099, eval/recall: 0.9006, eval/F1: 0.8998, lr: 0.0004, train/prefecth_time: 0.0040 BEST_EVAL_ACC: 0.9006, at 2000 iters\n", + "2050 iteration USE_EMA: True, train/sup_loss: 0.0362, train/unsup_loss: 0.0596, train/total_loss: 0.0959, train/util_ratio: 0.7812, train/run_time: 0.1412, lr: 0.0004, train/prefecth_time: 0.0040 \n", + "2100 iteration USE_EMA: True, train/sup_loss: 0.0052, train/unsup_loss: 0.2378, train/total_loss: 0.2430, train/util_ratio: 0.7500, train/run_time: 0.1412, lr: 0.0004, train/prefecth_time: 0.0039 \n", + "2150 iteration USE_EMA: True, train/sup_loss: 0.0005, train/unsup_loss: 0.1701, train/total_loss: 0.1707, train/util_ratio: 0.8125, train/run_time: 0.1414, lr: 0.0004, train/prefecth_time: 0.0036 \n", + "2200 iteration USE_EMA: True, train/sup_loss: 0.0006, train/unsup_loss: 0.0899, train/total_loss: 0.0905, train/util_ratio: 0.7500, train/run_time: 0.1408, lr: 0.0004, train/prefecth_time: 0.0036 \n", + "2250 iteration USE_EMA: True, train/sup_loss: 0.0319, train/unsup_loss: 0.1628, train/total_loss: 0.1947, train/util_ratio: 0.6562, train/run_time: 0.1414, lr: 0.0004, train/prefecth_time: 0.0034 \n", + "2300 iteration USE_EMA: True, train/sup_loss: 0.0542, train/unsup_loss: 0.0963, train/total_loss: 0.1504, train/util_ratio: 0.8125, train/run_time: 0.1405, lr: 0.0004, train/prefecth_time: 0.0035 \n", + "2350 iteration USE_EMA: True, train/sup_loss: 0.0023, train/unsup_loss: 0.0910, train/total_loss: 0.0934, train/util_ratio: 0.7812, train/run_time: 0.1412, lr: 0.0004, train/prefecth_time: 0.0038 \n", + "2400 iteration USE_EMA: True, train/sup_loss: 0.0260, train/unsup_loss: 0.0474, train/total_loss: 0.0733, train/util_ratio: 0.7188, train/run_time: 0.1413, lr: 0.0004, train/prefecth_time: 0.0037 \n", + "2450 iteration USE_EMA: True, train/sup_loss: 0.0037, train/unsup_loss: 0.1110, train/total_loss: 0.1147, train/util_ratio: 0.8125, train/run_time: 0.1419, lr: 0.0004, train/prefecth_time: 0.0035 \n", + "validating...\n", + "confusion matrix:\n", + "[[0.954 0.003 0.001 0.001 0. 0. 0.002 0. 0.021 0.018]\n", + " [0. 0.99 0. 0. 0. 0. 0. 0. 0. 0.01 ]\n", + " [0.042 0. 0.749 0.006 0.144 0.012 0.034 0.012 0. 0.001]\n", + " [0.006 0.001 0.001 0.826 0.03 0.096 0.027 0.007 0.003 0.003]\n", + " [0. 0. 0.001 0.006 0.954 0.003 0.006 0.029 0.001 0. ]\n", + " [0. 0. 0.001 0.021 0.009 0.947 0.004 0.018 0. 0. ]\n", + " [0.001 0. 0.003 0. 0.001 0.002 0.992 0. 0.001 0. ]\n", + " [0.007 0. 0.002 0.013 0.029 0.042 0. 0.907 0. 0. ]\n", + " [0.024 0.009 0. 0. 0.001 0. 0.001 0. 0.95 0.015]\n", + " [0.004 0.025 0. 0. 0. 0. 0. 0. 0.001 0.97 ]]\n", + "model saved: ./saved_models/fixmatch/latest_model.pth\n", + "model saved: ./saved_models/fixmatch/model_best.pth\n", + "2500 iteration, USE_EMA: True, train/sup_loss: 0.0010, train/unsup_loss: 0.1853, train/total_loss: 0.1862, train/util_ratio: 0.8750, train/run_time: 0.1411, eval/loss: 0.2508, eval/top-1-acc: 0.9239, eval/balanced_acc: 0.9239, eval/precision: 0.9281, eval/recall: 0.9239, eval/F1: 0.9230, lr: 0.0004, train/prefecth_time: 0.0039 BEST_EVAL_ACC: 0.9239, at 2500 iters\n", + "2550 iteration USE_EMA: True, train/sup_loss: 0.0608, train/unsup_loss: 0.1851, train/total_loss: 0.2459, train/util_ratio: 0.7188, train/run_time: 0.1410, lr: 0.0004, train/prefecth_time: 0.0036 \n", + "2600 iteration USE_EMA: True, train/sup_loss: 0.0012, train/unsup_loss: 0.1002, train/total_loss: 0.1014, train/util_ratio: 0.6875, train/run_time: 0.1412, lr: 0.0004, train/prefecth_time: 0.0039 \n", + "2650 iteration USE_EMA: True, train/sup_loss: 0.0130, train/unsup_loss: 0.0630, train/total_loss: 0.0761, train/util_ratio: 0.7500, train/run_time: 0.1410, lr: 0.0004, train/prefecth_time: 0.0035 \n", + "2700 iteration USE_EMA: True, train/sup_loss: 0.0062, train/unsup_loss: 0.1358, train/total_loss: 0.1421, train/util_ratio: 0.8750, train/run_time: 0.1408, lr: 0.0004, train/prefecth_time: 0.0039 \n", + "2750 iteration USE_EMA: True, train/sup_loss: 0.0571, train/unsup_loss: 0.1289, train/total_loss: 0.1860, train/util_ratio: 0.6875, train/run_time: 0.1406, lr: 0.0004, train/prefecth_time: 0.0035 \n", + "2800 iteration USE_EMA: True, train/sup_loss: 0.0753, train/unsup_loss: 0.1043, train/total_loss: 0.1796, train/util_ratio: 0.7812, train/run_time: 0.1414, lr: 0.0004, train/prefecth_time: 0.0039 \n", + "2850 iteration USE_EMA: True, train/sup_loss: 0.0027, train/unsup_loss: 0.0043, train/total_loss: 0.0070, train/util_ratio: 0.7188, train/run_time: 0.1410, lr: 0.0004, train/prefecth_time: 0.0037 \n", + "2900 iteration USE_EMA: True, train/sup_loss: 0.0065, train/unsup_loss: 0.1434, train/total_loss: 0.1499, train/util_ratio: 0.8438, train/run_time: 0.1423, lr: 0.0003, train/prefecth_time: 0.0036 \n", + "2950 iteration USE_EMA: True, train/sup_loss: 0.0813, train/unsup_loss: 0.3382, train/total_loss: 0.4195, train/util_ratio: 0.7500, train/run_time: 0.1412, lr: 0.0003, train/prefecth_time: 0.0037 \n", + "validating...\n", + "confusion matrix:\n", + "[[0.958 0.002 0.001 0.001 0. 0. 0.002 0. 0.022 0.014]\n", + " [0. 0.989 0. 0. 0. 0. 0. 0. 0.001 0.01 ]\n", + " [0.042 0. 0.764 0.007 0.14 0.012 0.024 0.01 0. 0.001]\n", + " [0.005 0.001 0.003 0.841 0.019 0.095 0.023 0.007 0.003 0.003]\n", + " [0. 0. 0.002 0.009 0.941 0.006 0.007 0.034 0.001 0. ]\n", + " [0. 0. 0.001 0.018 0.007 0.956 0.002 0.016 0. 0. ]\n", + " [0.001 0. 0.003 0. 0. 0.002 0.993 0. 0.001 0. ]\n", + " [0.005 0. 0.002 0.01 0.012 0.047 0. 0.924 0. 0. ]\n", + " [0.016 0.009 0. 0. 0. 0. 0.001 0. 0.964 0.01 ]\n", + " [0.004 0.027 0. 0. 0. 0. 0. 0. 0.001 0.968]]\n", + "model saved: ./saved_models/fixmatch/latest_model.pth\n", + "model saved: ./saved_models/fixmatch/model_best.pth\n", + "3000 iteration, USE_EMA: True, train/sup_loss: 0.0019, train/unsup_loss: 0.0714, train/total_loss: 0.0733, train/util_ratio: 0.7188, train/run_time: 0.1415, eval/loss: 0.2337, eval/top-1-acc: 0.9298, eval/balanced_acc: 0.9298, eval/precision: 0.9330, eval/recall: 0.9298, eval/F1: 0.9290, lr: 0.0003, train/prefecth_time: 0.0036 BEST_EVAL_ACC: 0.9298, at 3000 iters\n", + "3050 iteration USE_EMA: True, train/sup_loss: 0.0003, train/unsup_loss: 0.2654, train/total_loss: 0.2657, train/util_ratio: 0.6562, train/run_time: 0.1415, lr: 0.0003, train/prefecth_time: 0.0039 \n", + "3100 iteration USE_EMA: True, train/sup_loss: 0.0526, train/unsup_loss: 0.0743, train/total_loss: 0.1269, train/util_ratio: 0.7812, train/run_time: 0.1411, lr: 0.0003, train/prefecth_time: 0.0035 \n", + "3150 iteration USE_EMA: True, train/sup_loss: 0.0004, train/unsup_loss: 0.1189, train/total_loss: 0.1193, train/util_ratio: 0.8750, train/run_time: 0.1406, lr: 0.0003, train/prefecth_time: 0.0038 \n", + "3200 iteration USE_EMA: True, train/sup_loss: 0.0074, train/unsup_loss: 0.1950, train/total_loss: 0.2024, train/util_ratio: 0.6562, train/run_time: 0.1411, lr: 0.0003, train/prefecth_time: 0.0036 \n", + "3250 iteration USE_EMA: True, train/sup_loss: 0.0459, train/unsup_loss: 0.3374, train/total_loss: 0.3833, train/util_ratio: 0.8438, train/run_time: 0.1411, lr: 0.0003, train/prefecth_time: 0.0037 \n", + "3300 iteration USE_EMA: True, train/sup_loss: 0.0059, train/unsup_loss: 0.2219, train/total_loss: 0.2278, train/util_ratio: 0.8438, train/run_time: 0.1417, lr: 0.0003, train/prefecth_time: 0.0037 \n", + "3350 iteration USE_EMA: True, train/sup_loss: 0.0015, train/unsup_loss: 0.0706, train/total_loss: 0.0720, train/util_ratio: 0.7812, train/run_time: 0.1413, lr: 0.0003, train/prefecth_time: 0.0036 \n", + "3400 iteration USE_EMA: True, train/sup_loss: 0.2013, train/unsup_loss: 0.0645, train/total_loss: 0.2658, train/util_ratio: 0.7812, train/run_time: 0.1415, lr: 0.0003, train/prefecth_time: 0.0038 \n", + "3450 iteration USE_EMA: True, train/sup_loss: 0.0029, train/unsup_loss: 0.3704, train/total_loss: 0.3733, train/util_ratio: 0.9062, train/run_time: 0.1414, lr: 0.0003, train/prefecth_time: 0.0037 \n", + "validating...\n", + "confusion matrix:\n", + "[[0.957 0.002 0.002 0.001 0. 0. 0.001 0. 0.022 0.015]\n", + " [0. 0.988 0. 0. 0. 0. 0. 0. 0.001 0.011]\n", + " [0.043 0. 0.774 0.007 0.134 0.011 0.021 0.009 0. 0.001]\n", + " [0.003 0.001 0.003 0.862 0.016 0.083 0.023 0.004 0.003 0.002]\n", + " [0. 0. 0.003 0.01 0.936 0.007 0.008 0.035 0.001 0. ]\n", + " [0. 0. 0.001 0.022 0.006 0.955 0.002 0.014 0. 0. ]\n", + " [0.001 0. 0.004 0.001 0. 0.002 0.991 0. 0.001 0. ]\n", + " [0.005 0. 0.002 0.009 0.006 0.046 0. 0.932 0. 0. ]\n", + " [0.014 0.007 0. 0. 0. 0. 0.001 0. 0.969 0.009]\n", + " [0.004 0.028 0. 0. 0. 0. 0. 0. 0.002 0.966]]\n", + "model saved: ./saved_models/fixmatch/latest_model.pth\n", + "model saved: ./saved_models/fixmatch/model_best.pth\n", + "3500 iteration, USE_EMA: True, train/sup_loss: 0.0351, train/unsup_loss: 0.2334, train/total_loss: 0.2685, train/util_ratio: 0.8750, train/run_time: 0.1405, eval/loss: 0.2236, eval/top-1-acc: 0.9330, eval/balanced_acc: 0.9330, eval/precision: 0.9355, eval/recall: 0.9330, eval/F1: 0.9323, lr: 0.0003, train/prefecth_time: 0.0037 BEST_EVAL_ACC: 0.9330, at 3500 iters\n", + "3550 iteration USE_EMA: True, train/sup_loss: 0.0035, train/unsup_loss: 0.1521, train/total_loss: 0.1556, train/util_ratio: 0.7500, train/run_time: 0.1413, lr: 0.0003, train/prefecth_time: 0.0038 \n", + "3600 iteration USE_EMA: True, train/sup_loss: 0.0002, train/unsup_loss: 0.3567, train/total_loss: 0.3570, train/util_ratio: 0.8125, train/run_time: 0.1416, lr: 0.0003, train/prefecth_time: 0.0040 \n", + "3650 iteration USE_EMA: True, train/sup_loss: 0.0004, train/unsup_loss: 0.1816, train/total_loss: 0.1820, train/util_ratio: 0.6875, train/run_time: 0.1410, lr: 0.0003, train/prefecth_time: 0.0039 \n", + "3700 iteration USE_EMA: True, train/sup_loss: 0.0007, train/unsup_loss: 0.3171, train/total_loss: 0.3178, train/util_ratio: 0.7500, train/run_time: 0.1409, lr: 0.0003, train/prefecth_time: 0.0039 \n", + "3750 iteration USE_EMA: True, train/sup_loss: 0.0019, train/unsup_loss: 0.2061, train/total_loss: 0.2079, train/util_ratio: 0.8750, train/run_time: 0.1410, lr: 0.0003, train/prefecth_time: 0.0038 \n", + "3800 iteration USE_EMA: True, train/sup_loss: 0.0040, train/unsup_loss: 0.0473, train/total_loss: 0.0514, train/util_ratio: 0.8750, train/run_time: 0.1416, lr: 0.0003, train/prefecth_time: 0.0037 \n", + "3850 iteration USE_EMA: True, train/sup_loss: 0.0848, train/unsup_loss: 0.0998, train/total_loss: 0.1846, train/util_ratio: 0.9375, train/run_time: 0.1414, lr: 0.0002, train/prefecth_time: 0.0036 \n", + "3900 iteration USE_EMA: True, train/sup_loss: 0.0017, train/unsup_loss: 0.0173, train/total_loss: 0.0190, train/util_ratio: 0.7812, train/run_time: 0.1416, lr: 0.0002, train/prefecth_time: 0.0036 \n", + "3950 iteration USE_EMA: True, train/sup_loss: 0.0003, train/unsup_loss: 0.0839, train/total_loss: 0.0842, train/util_ratio: 0.6875, train/run_time: 0.1406, lr: 0.0002, train/prefecth_time: 0.0038 \n", + "validating...\n", + "confusion matrix:\n", + "[[0.957 0.002 0.002 0.001 0. 0. 0.001 0. 0.022 0.015]\n", + " [0. 0.988 0. 0. 0. 0. 0. 0. 0.001 0.011]\n", + " [0.041 0. 0.782 0.006 0.132 0.01 0.019 0.009 0. 0.001]\n", + " [0.003 0.001 0.003 0.861 0.019 0.084 0.022 0.002 0.003 0.002]\n", + " [0. 0. 0.003 0.008 0.938 0.008 0.007 0.035 0.001 0. ]\n", + " [0. 0. 0.001 0.021 0.006 0.956 0.002 0.014 0. 0. ]\n", + " [0.002 0. 0.003 0.001 0. 0.002 0.991 0. 0.001 0. ]\n", + " [0.004 0. 0.002 0.008 0.006 0.039 0. 0.94 0.001 0. ]\n", + " [0.011 0.006 0. 0. 0. 0. 0. 0. 0.972 0.011]\n", + " [0.003 0.03 0. 0. 0. 0. 0. 0. 0.002 0.965]]\n", "model saved: ./saved_models/fixmatch/latest_model.pth\n", - "model saved: ./saved_models/fixmatch/model_best.pth\n" + "model saved: ./saved_models/fixmatch/model_best.pth\n", + "4000 iteration, USE_EMA: True, train/sup_loss: 0.0075, train/unsup_loss: 0.1623, train/total_loss: 0.1698, train/util_ratio: 0.9062, train/run_time: 0.1419, eval/loss: 0.2220, eval/top-1-acc: 0.9350, eval/balanced_acc: 0.9350, eval/precision: 0.9374, eval/recall: 0.9350, eval/F1: 0.9344, lr: 0.0002, train/prefecth_time: 0.0038 BEST_EVAL_ACC: 0.9350, at 4000 iters\n", + "4050 iteration USE_EMA: True, train/sup_loss: 0.0002, train/unsup_loss: 0.1693, train/total_loss: 0.1695, train/util_ratio: 0.6250, train/run_time: 0.1410, lr: 0.0002, train/prefecth_time: 0.0031 \n", + "4100 iteration USE_EMA: True, train/sup_loss: 0.0013, train/unsup_loss: 0.0134, train/total_loss: 0.0147, train/util_ratio: 0.6875, train/run_time: 0.1410, lr: 0.0002, train/prefecth_time: 0.0039 \n", + "4150 iteration USE_EMA: True, train/sup_loss: 0.0192, train/unsup_loss: 0.0331, train/total_loss: 0.0522, train/util_ratio: 0.5938, train/run_time: 0.1412, lr: 0.0002, train/prefecth_time: 0.0039 \n", + "4200 iteration USE_EMA: True, train/sup_loss: 0.0165, train/unsup_loss: 0.1861, train/total_loss: 0.2026, train/util_ratio: 0.7812, train/run_time: 0.1409, lr: 0.0002, train/prefecth_time: 0.0038 \n", + "4250 iteration USE_EMA: True, train/sup_loss: 0.0276, train/unsup_loss: 0.1913, train/total_loss: 0.2189, train/util_ratio: 0.8750, train/run_time: 0.1412, lr: 0.0002, train/prefecth_time: 0.0036 \n", + "4300 iteration USE_EMA: True, train/sup_loss: 0.0003, train/unsup_loss: 0.1182, train/total_loss: 0.1186, train/util_ratio: 0.6562, train/run_time: 0.1407, lr: 0.0002, train/prefecth_time: 0.0036 \n", + "4350 iteration USE_EMA: True, train/sup_loss: 0.0138, train/unsup_loss: 0.2013, train/total_loss: 0.2152, train/util_ratio: 0.7188, train/run_time: 0.1411, lr: 0.0002, train/prefecth_time: 0.0039 \n", + "4400 iteration USE_EMA: True, train/sup_loss: 0.0021, train/unsup_loss: 0.0658, train/total_loss: 0.0679, train/util_ratio: 0.8438, train/run_time: 0.1405, lr: 0.0002, train/prefecth_time: 0.0036 \n", + "4450 iteration USE_EMA: True, train/sup_loss: 0.0002, train/unsup_loss: 0.1821, train/total_loss: 0.1823, train/util_ratio: 0.6562, train/run_time: 0.1410, lr: 0.0002, train/prefecth_time: 0.0037 \n", + "validating...\n", + "confusion matrix:\n", + "[[0.957 0.002 0.002 0.001 0. 0. 0. 0. 0.022 0.016]\n", + " [0. 0.989 0. 0. 0. 0. 0. 0. 0.001 0.01 ]\n", + " [0.042 0. 0.782 0.006 0.134 0.01 0.019 0.006 0. 0.001]\n", + " [0.003 0.001 0.003 0.861 0.018 0.086 0.022 0.001 0.003 0.002]\n", + " [0. 0. 0.003 0.007 0.943 0.008 0.007 0.031 0.001 0. ]\n", + " [0. 0. 0.001 0.019 0.006 0.96 0.002 0.012 0. 0. ]\n", + " [0.002 0. 0.003 0.001 0. 0.002 0.991 0. 0.001 0. ]\n", + " [0.004 0. 0.002 0.007 0.006 0.039 0. 0.941 0.001 0. ]\n", + " [0.008 0.007 0. 0. 0. 0. 0. 0. 0.975 0.01 ]\n", + " [0.003 0.031 0. 0. 0. 0. 0. 0. 0.001 0.965]]\n", + "model saved: ./saved_models/fixmatch/latest_model.pth\n", + "model saved: ./saved_models/fixmatch/model_best.pth\n", + "4500 iteration, USE_EMA: True, train/sup_loss: 0.0618, train/unsup_loss: 0.2132, train/total_loss: 0.2750, train/util_ratio: 0.8750, train/run_time: 0.1411, eval/loss: 0.2234, eval/top-1-acc: 0.9364, eval/balanced_acc: 0.9364, eval/precision: 0.9390, eval/recall: 0.9364, eval/F1: 0.9358, lr: 0.0002, train/prefecth_time: 0.0038 BEST_EVAL_ACC: 0.9364, at 4500 iters\n", + "4550 iteration USE_EMA: True, train/sup_loss: 0.0178, train/unsup_loss: 0.0903, train/total_loss: 0.1081, train/util_ratio: 0.8438, train/run_time: 0.1419, lr: 0.0002, train/prefecth_time: 0.0038 \n", + "4600 iteration USE_EMA: True, train/sup_loss: 0.0589, train/unsup_loss: 0.3741, train/total_loss: 0.4330, train/util_ratio: 0.6875, train/run_time: 0.1414, lr: 0.0002, train/prefecth_time: 0.0036 \n", + "4650 iteration USE_EMA: True, train/sup_loss: 0.0009, train/unsup_loss: 0.0721, train/total_loss: 0.0730, train/util_ratio: 0.7500, train/run_time: 0.1405, lr: 0.0001, train/prefecth_time: 0.0035 \n", + "4700 iteration USE_EMA: True, train/sup_loss: 0.0007, train/unsup_loss: 0.2161, train/total_loss: 0.2168, train/util_ratio: 0.8750, train/run_time: 0.1413, lr: 0.0001, train/prefecth_time: 0.0040 \n", + "4750 iteration USE_EMA: True, train/sup_loss: 0.0015, train/unsup_loss: 0.1537, train/total_loss: 0.1552, train/util_ratio: 0.6875, train/run_time: 0.1407, lr: 0.0001, train/prefecth_time: 0.0039 \n", + "4800 iteration USE_EMA: True, train/sup_loss: 0.0409, train/unsup_loss: 0.2510, train/total_loss: 0.2919, train/util_ratio: 0.8125, train/run_time: 0.1408, lr: 0.0001, train/prefecth_time: 0.0036 \n", + "4850 iteration USE_EMA: True, train/sup_loss: 0.0002, train/unsup_loss: 0.2873, train/total_loss: 0.2875, train/util_ratio: 0.7812, train/run_time: 0.1418, lr: 0.0001, train/prefecth_time: 0.0039 \n", + "4900 iteration USE_EMA: True, train/sup_loss: 0.0004, train/unsup_loss: 0.0784, train/total_loss: 0.0788, train/util_ratio: 0.7812, train/run_time: 0.1412, lr: 0.0001, train/prefecth_time: 0.0037 \n", + "4950 iteration USE_EMA: True, train/sup_loss: 0.0006, train/unsup_loss: 0.1122, train/total_loss: 0.1128, train/util_ratio: 0.7812, train/run_time: 0.1413, lr: 0.0001, train/prefecth_time: 0.0039 \n", + "validating...\n", + "confusion matrix:\n", + "[[0.957 0.002 0.002 0.001 0. 0. 0. 0. 0.022 0.016]\n", + " [0. 0.989 0. 0. 0. 0. 0. 0. 0.001 0.01 ]\n", + " [0.043 0. 0.783 0.007 0.132 0.01 0.019 0.005 0. 0.001]\n", + " [0.003 0.001 0.003 0.871 0.016 0.077 0.024 0.001 0.002 0.002]\n", + " [0. 0. 0.003 0.007 0.947 0.007 0.007 0.028 0.001 0. ]\n", + " [0. 0. 0.001 0.019 0.006 0.96 0.002 0.012 0. 0. ]\n", + " [0.002 0. 0.003 0. 0. 0.002 0.992 0. 0.001 0. ]\n", + " [0.005 0. 0.002 0.005 0.006 0.032 0. 0.948 0.001 0.001]\n", + " [0.006 0.008 0. 0. 0. 0. 0. 0. 0.977 0.009]\n", + " [0.003 0.033 0. 0. 0. 0. 0. 0. 0.001 0.963]]\n", + "model saved: ./saved_models/fixmatch/latest_model.pth\n", + "model saved: ./saved_models/fixmatch/model_best.pth\n", + "5000 iteration, USE_EMA: True, train/sup_loss: 0.0005, train/unsup_loss: 0.2451, train/total_loss: 0.2457, train/util_ratio: 0.7500, train/run_time: 0.1409, eval/loss: 0.2223, eval/top-1-acc: 0.9387, eval/balanced_acc: 0.9387, eval/precision: 0.9410, eval/recall: 0.9387, eval/F1: 0.9380, lr: 0.0001, train/prefecth_time: 0.0034 BEST_EVAL_ACC: 0.9387, at 5000 iters\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[2023-07-19 23:13:01,538 INFO] confusion matrix\n", + "[2023-07-19 23:13:01,540 INFO] [[0.956 0.004 0. 0.001 0. 0. 0. 0. 0.022 0.017]\n", + " [0. 0.99 0. 0. 0. 0. 0. 0. 0. 0.01 ]\n", + " [0.046 0. 0.77 0.01 0.134 0.016 0.018 0.004 0.001 0.001]\n", + " [0.003 0.002 0.002 0.872 0.015 0.079 0.023 0. 0.002 0.002]\n", + " [0. 0. 0.002 0.007 0.949 0.007 0.008 0.026 0.001 0. ]\n", + " [0. 0. 0.001 0.018 0.005 0.965 0.002 0.009 0. 0. ]\n", + " [0.002 0. 0.003 0.001 0. 0.002 0.991 0. 0.001 0. ]\n", + " [0.004 0. 0.002 0.005 0.005 0.03 0. 0.951 0.001 0.002]\n", + " [0.006 0.008 0. 0. 0. 0. 0. 0. 0.977 0.009]\n", + " [0.003 0.034 0. 0. 0. 0. 0. 0. 0.001 0.962]]\n", + "[2023-07-19 23:13:01,542 INFO] evaluation metric\n", + "[2023-07-19 23:13:01,543 INFO] acc: 0.9383\n", + "[2023-07-19 23:13:01,543 INFO] precision: 0.9409\n", + "[2023-07-19 23:13:01,544 INFO] recall: 0.9383\n", + "[2023-07-19 23:13:01,544 INFO] f1: 0.9375\n", + "[2023-07-19 23:13:02,617 INFO] Best acc 0.9387 at epoch 0\n", + "[2023-07-19 23:13:02,618 INFO] Training finished.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "model saved: ./saved_models/fixmatch/latest_model.pth\n" ] } ], @@ -294,30 +562,31 @@ "name": "stderr", "output_type": "stream", "text": [ - "/home/haoc/miniconda3/envs/test/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "[2022-12-18 14:02:22,466 INFO] confusion matrix\n", - "[2022-12-18 14:02:22,468 INFO] [[0. 0. 1. 0. 0. 0. 0. 0. 0. 0.]\n", - " [0. 0. 1. 0. 0. 0. 0. 0. 0. 0.]\n", - " [0. 0. 1. 0. 0. 0. 0. 0. 0. 0.]\n", - " [0. 0. 1. 0. 0. 0. 0. 0. 0. 0.]\n", - " [0. 0. 1. 0. 0. 0. 0. 0. 0. 0.]\n", - " [0. 0. 1. 0. 0. 0. 0. 0. 0. 0.]\n", - " [0. 0. 1. 0. 0. 0. 0. 0. 0. 0.]\n", - " [0. 0. 1. 0. 0. 0. 0. 0. 0. 0.]\n", - " [0. 0. 1. 0. 0. 0. 0. 0. 0. 0.]\n", - " [0. 0. 1. 0. 0. 0. 0. 0. 0. 0.]]\n", - "[2022-12-18 14:02:22,470 INFO] evaluation metric\n", - "[2022-12-18 14:02:22,471 INFO] acc: 0.1000\n", - "[2022-12-18 14:02:22,472 INFO] precision: 0.0100\n", - "[2022-12-18 14:02:22,474 INFO] recall: 0.1000\n", - "[2022-12-18 14:02:22,475 INFO] f1: 0.0182\n" + "[2023-07-19 23:14:07,511 INFO] confusion matrix\n", + "[2023-07-19 23:14:07,513 INFO] [[0.956 0.004 0. 0.001 0. 0. 0. 0. 0.022 0.017]\n", + " [0. 0.99 0. 0. 0. 0. 0. 0. 0. 0.01 ]\n", + " [0.046 0. 0.77 0.01 0.134 0.016 0.018 0.004 0.001 0.001]\n", + " [0.003 0.002 0.002 0.872 0.015 0.079 0.023 0. 0.002 0.002]\n", + " [0. 0. 0.002 0.007 0.949 0.007 0.008 0.026 0.001 0. ]\n", + " [0. 0. 0.001 0.018 0.005 0.965 0.002 0.009 0. 0. ]\n", + " [0.002 0. 0.003 0.001 0. 0.002 0.991 0. 0.001 0. ]\n", + " [0.004 0. 0.002 0.005 0.005 0.03 0. 0.951 0.001 0.002]\n", + " [0.006 0.008 0. 0. 0. 0. 0. 0. 0.977 0.009]\n", + " [0.003 0.034 0. 0. 0. 0. 0. 0. 0.001 0.962]]\n", + "[2023-07-19 23:14:07,515 INFO] evaluation metric\n", + "[2023-07-19 23:14:07,516 INFO] acc: 0.9383\n", + "[2023-07-19 23:14:07,517 INFO] precision: 0.9409\n", + "[2023-07-19 23:14:07,517 INFO] recall: 0.9383\n", + "[2023-07-19 23:14:07,518 INFO] f1: 0.9375\n" ] }, { "data": { "text/plain": [ - "{'acc': 0.1, 'precision': 0.01, 'recall': 0.1, 'f1': 0.01818181818181818}" + "{'acc': 0.9383,\n", + " 'precision': 0.9408766240300274,\n", + " 'recall': 0.9382999999999999,\n", + " 'f1': 0.9375322532701483}" ] }, "execution_count": 6, @@ -350,16 +619,7 @@ "name": "#%%\n" } }, - "outputs": [ - { - "ename": "", - "evalue": "", - "output_type": "error", - "traceback": [ - "\u001b[1;31mThe Kernel crashed while executing code in the the current cell or a previous cell. Please review the code in the cell(s) to identify a possible cause of the failure. Click here for more info. View Jupyter log for further details." - ] - } - ], + "outputs": [], "source": [ "y_pred, y_logits = trainer.predict(eval_loader)" ] @@ -381,7 +641,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.13" + "version": "3.9.17" }, "vscode": { "interpreter": { diff --git a/notebooks/Custom_Algorithm.ipynb b/notebooks/Custom_Algorithm.ipynb index 9d8853dbc..f4cfedfce 100644 --- a/notebooks/Custom_Algorithm.ipynb +++ b/notebooks/Custom_Algorithm.ipynb @@ -20,7 +20,16 @@ "cell_type": "code", "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/media/Zeus/haoc/miniconda/envs/test_semilearn_031/lib/python3.9/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], "source": [ "from semilearn import get_dataset, get_data_loader, get_net_builder, get_algorithm, get_config, Trainer\n" ] @@ -148,13 +157,6 @@ "execution_count": 3, "metadata": {}, "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "here 1\n" - ] - }, { "name": "stderr", "output_type": "stream", @@ -278,31 +280,297 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch: 0\n" + "Epoch: 0\n", + "50 iteration USE_EMA: True, train/sup_loss: 3.5051, train/unsup_loss: 0.0000, train/total_loss: 3.5051, train/util_ratio: 0.0000, train/run_time: 0.1370, lr: 0.0005, train/prefecth_time: 0.0036 \n", + "100 iteration USE_EMA: True, train/sup_loss: 1.0864, train/unsup_loss: 0.0000, train/total_loss: 1.0864, train/util_ratio: 0.0000, train/run_time: 0.1388, lr: 0.0005, train/prefecth_time: 0.0031 \n", + "150 iteration USE_EMA: True, train/sup_loss: 0.5159, train/unsup_loss: 0.0000, train/total_loss: 0.5159, train/util_ratio: 0.0000, train/run_time: 0.1384, lr: 0.0005, train/prefecth_time: 0.0035 \n", + "200 iteration USE_EMA: True, train/sup_loss: 0.5048, train/unsup_loss: 0.0000, train/total_loss: 0.5048, train/util_ratio: 0.0000, train/run_time: 0.1384, lr: 0.0005, train/prefecth_time: 0.0037 \n", + "250 iteration USE_EMA: True, train/sup_loss: 0.5692, train/unsup_loss: 0.1162, train/total_loss: 0.6854, train/util_ratio: 0.1562, train/run_time: 0.1395, lr: 0.0005, train/prefecth_time: 0.0037 \n", + "300 iteration USE_EMA: True, train/sup_loss: 0.1154, train/unsup_loss: 0.0059, train/total_loss: 0.1214, train/util_ratio: 0.0625, train/run_time: 0.1391, lr: 0.0005, train/prefecth_time: 0.0035 \n", + "350 iteration USE_EMA: True, train/sup_loss: 0.0512, train/unsup_loss: 0.0717, train/total_loss: 0.1230, train/util_ratio: 0.0938, train/run_time: 0.1396, lr: 0.0005, train/prefecth_time: 0.0036 \n", + "400 iteration USE_EMA: True, train/sup_loss: 0.1361, train/unsup_loss: 0.0063, train/total_loss: 0.1425, train/util_ratio: 0.1250, train/run_time: 0.1401, lr: 0.0005, train/prefecth_time: 0.0035 \n", + "450 iteration USE_EMA: True, train/sup_loss: 0.0383, train/unsup_loss: 0.0889, train/total_loss: 0.1272, train/util_ratio: 0.3125, train/run_time: 0.1403, lr: 0.0005, train/prefecth_time: 0.0037 \n", + "validating...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/media/Zeus/haoc/miniconda/envs/test_semilearn_031/lib/python3.9/site-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "confusion matrix:\n", + "[[0. 0. 0. 0. 0. 0. 0. 1. 0. 0. ]\n", + " [0. 0. 0. 0. 0. 0. 0. 1. 0. 0. ]\n", + " [0. 0. 0. 0. 0. 0. 0. 1. 0. 0. ]\n", + " [0. 0. 0. 0.004 0. 0.001 0. 0.995 0. 0. ]\n", + " [0. 0. 0. 0.002 0. 0. 0. 0.998 0. 0. ]\n", + " [0. 0. 0. 0.001 0. 0.014 0. 0.985 0. 0. ]\n", + " [0. 0. 0. 0. 0. 0. 0. 1. 0. 0. ]\n", + " [0. 0. 0. 0. 0. 0. 0. 1. 0. 0. ]\n", + " [0. 0. 0. 0. 0. 0. 0. 1. 0. 0. ]\n", + " [0. 0. 0. 0. 0. 0. 0. 1. 0. 0. ]]\n", + "model saved: ./saved_models/fixmatch/latest_model.pth\n", + "model saved: ./saved_models/fixmatch/model_best.pth\n", + "500 iteration, USE_EMA: True, train/sup_loss: 0.0129, train/unsup_loss: 0.0249, train/total_loss: 0.0378, train/util_ratio: 0.1250, train/run_time: 0.1401, eval/loss: 5.5155, eval/top-1-acc: 0.1018, eval/balanced_acc: 0.1018, eval/precision: 0.1605, eval/recall: 0.1018, eval/F1: 0.0218, lr: 0.0005, train/prefecth_time: 0.0035 BEST_EVAL_ACC: 0.1018, at 500 iters\n", + "550 iteration USE_EMA: True, train/sup_loss: 0.0389, train/unsup_loss: 0.1663, train/total_loss: 0.2052, train/util_ratio: 0.3125, train/run_time: 0.1406, lr: 0.0005, train/prefecth_time: 0.0039 \n", + "600 iteration USE_EMA: True, train/sup_loss: 0.0373, train/unsup_loss: 0.1409, train/total_loss: 0.1782, train/util_ratio: 0.3438, train/run_time: 0.1402, lr: 0.0005, train/prefecth_time: 0.0038 \n", + "650 iteration USE_EMA: True, train/sup_loss: 0.0120, train/unsup_loss: 0.4798, train/total_loss: 0.4917, train/util_ratio: 0.4062, train/run_time: 0.1406, lr: 0.0005, train/prefecth_time: 0.0037 \n", + "700 iteration USE_EMA: True, train/sup_loss: 0.0063, train/unsup_loss: 0.1671, train/total_loss: 0.1734, train/util_ratio: 0.3438, train/run_time: 0.1410, lr: 0.0005, train/prefecth_time: 0.0036 \n", + "750 iteration USE_EMA: True, train/sup_loss: 0.1073, train/unsup_loss: 0.4499, train/total_loss: 0.5573, train/util_ratio: 0.4688, train/run_time: 0.1413, lr: 0.0005, train/prefecth_time: 0.0040 \n", + "800 iteration USE_EMA: True, train/sup_loss: 0.0048, train/unsup_loss: 0.3912, train/total_loss: 0.3960, train/util_ratio: 0.5938, train/run_time: 0.1408, lr: 0.0005, train/prefecth_time: 0.0035 \n", + "850 iteration USE_EMA: True, train/sup_loss: 0.0266, train/unsup_loss: 0.3070, train/total_loss: 0.3336, train/util_ratio: 0.5000, train/run_time: 0.1407, lr: 0.0005, train/prefecth_time: 0.0039 \n", + "900 iteration USE_EMA: True, train/sup_loss: 0.0046, train/unsup_loss: 0.0333, train/total_loss: 0.0378, train/util_ratio: 0.4375, train/run_time: 0.1407, lr: 0.0005, train/prefecth_time: 0.0039 \n", + "950 iteration USE_EMA: True, train/sup_loss: 0.0058, train/unsup_loss: 0.0415, train/total_loss: 0.0473, train/util_ratio: 0.4375, train/run_time: 0.1401, lr: 0.0005, train/prefecth_time: 0.0038 \n", + "validating...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/media/Zeus/haoc/miniconda/envs/test_semilearn_031/lib/python3.9/site-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", + " _warn_prf(average, modifier, msg_start, len(result))\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "confusion matrix:\n", + "[[0.303 0. 0.006 0.004 0. 0.003 0. 0.645 0. 0.039]\n", + " [0. 0.001 0. 0. 0. 0.001 0. 0.292 0. 0.706]\n", + " [0. 0. 0.506 0.018 0. 0.11 0. 0.361 0. 0.005]\n", + " [0. 0. 0.001 0.495 0. 0.263 0. 0.239 0. 0.002]\n", + " [0.001 0. 0.005 0.061 0. 0.123 0. 0.809 0. 0.001]\n", + " [0. 0. 0. 0.031 0. 0.78 0. 0.189 0. 0. ]\n", + " [0.001 0. 0.028 0.064 0. 0.231 0.002 0.67 0. 0.004]\n", + " [0. 0. 0. 0.004 0. 0.127 0. 0.868 0. 0.001]\n", + " [0.036 0. 0.004 0.004 0. 0.031 0. 0.677 0. 0.248]\n", + " [0. 0. 0. 0.002 0. 0.001 0. 0.121 0. 0.876]]\n", + "model saved: ./saved_models/fixmatch/latest_model.pth\n", + "model saved: ./saved_models/fixmatch/model_best.pth\n", + "1000 iteration, USE_EMA: True, train/sup_loss: 0.0189, train/unsup_loss: 0.2167, train/total_loss: 0.2356, train/util_ratio: 0.5000, train/run_time: 0.1406, eval/loss: 2.4450, eval/top-1-acc: 0.3831, eval/balanced_acc: 0.3831, eval/precision: 0.5644, eval/recall: 0.3831, eval/F1: 0.3187, lr: 0.0005, train/prefecth_time: 0.0036 BEST_EVAL_ACC: 0.3831, at 1000 iters\n", + "1050 iteration USE_EMA: True, train/sup_loss: 0.0346, train/unsup_loss: 0.0441, train/total_loss: 0.0788, train/util_ratio: 0.6875, train/run_time: 0.1408, lr: 0.0005, train/prefecth_time: 0.0038 \n", + "1100 iteration USE_EMA: True, train/sup_loss: 0.0026, train/unsup_loss: 0.3319, train/total_loss: 0.3345, train/util_ratio: 0.5938, train/run_time: 0.1411, lr: 0.0005, train/prefecth_time: 0.0035 \n", + "1150 iteration USE_EMA: True, train/sup_loss: 0.1744, train/unsup_loss: 0.3718, train/total_loss: 0.5462, train/util_ratio: 0.5312, train/run_time: 0.1416, lr: 0.0005, train/prefecth_time: 0.0040 \n", + "1200 iteration USE_EMA: True, train/sup_loss: 0.0111, train/unsup_loss: 0.1394, train/total_loss: 0.1504, train/util_ratio: 0.6250, train/run_time: 0.1416, lr: 0.0005, train/prefecth_time: 0.0040 \n", + "1250 iteration USE_EMA: True, train/sup_loss: 0.0191, train/unsup_loss: 0.2628, train/total_loss: 0.2818, train/util_ratio: 0.6250, train/run_time: 0.1412, lr: 0.0005, train/prefecth_time: 0.0036 \n", + "1300 iteration USE_EMA: True, train/sup_loss: 0.0078, train/unsup_loss: 0.1025, train/total_loss: 0.1103, train/util_ratio: 0.5625, train/run_time: 0.1416, lr: 0.0005, train/prefecth_time: 0.0038 \n", + "1350 iteration USE_EMA: True, train/sup_loss: 0.0248, train/unsup_loss: 0.6699, train/total_loss: 0.6947, train/util_ratio: 0.6562, train/run_time: 0.1408, lr: 0.0005, train/prefecth_time: 0.0039 \n", + "1400 iteration USE_EMA: True, train/sup_loss: 0.0063, train/unsup_loss: 0.3722, train/total_loss: 0.3785, train/util_ratio: 0.7188, train/run_time: 0.1411, lr: 0.0005, train/prefecth_time: 0.0034 \n", + "1450 iteration USE_EMA: True, train/sup_loss: 0.0029, train/unsup_loss: 0.0657, train/total_loss: 0.0686, train/util_ratio: 0.6875, train/run_time: 0.1409, lr: 0.0005, train/prefecth_time: 0.0038 \n", + "validating...\n", + "confusion matrix:\n", + "[[0.956 0. 0.003 0.002 0. 0.002 0.002 0.011 0.001 0.023]\n", + " [0. 0.88 0. 0. 0. 0. 0. 0. 0. 0.12 ]\n", + " [0.046 0. 0.669 0.017 0.003 0.117 0.005 0.137 0. 0.006]\n", + " [0.002 0.001 0.009 0.606 0.002 0.288 0.011 0.077 0.001 0.003]\n", + " [0.018 0. 0.012 0.039 0.387 0.081 0.003 0.46 0. 0. ]\n", + " [0. 0. 0. 0.044 0. 0.868 0. 0.088 0. 0. ]\n", + " [0.007 0. 0.023 0.005 0. 0.026 0.918 0.019 0. 0.002]\n", + " [0.001 0. 0.001 0.004 0.006 0.15 0. 0.836 0. 0.002]\n", + " [0.293 0.004 0.005 0.006 0. 0.012 0. 0.022 0.279 0.379]\n", + " [0.009 0.003 0. 0.001 0. 0. 0. 0. 0. 0.987]]\n", + "model saved: ./saved_models/fixmatch/latest_model.pth\n", + "model saved: ./saved_models/fixmatch/model_best.pth\n", + "1500 iteration, USE_EMA: True, train/sup_loss: 0.0016, train/unsup_loss: 0.1244, train/total_loss: 0.1260, train/util_ratio: 0.5938, train/run_time: 0.1409, eval/loss: 0.7701, eval/top-1-acc: 0.7386, eval/balanced_acc: 0.7386, eval/precision: 0.8133, eval/recall: 0.7386, eval/F1: 0.7264, lr: 0.0005, train/prefecth_time: 0.0034 BEST_EVAL_ACC: 0.7386, at 1500 iters\n", + "1550 iteration USE_EMA: True, train/sup_loss: 0.0057, train/unsup_loss: 0.2125, train/total_loss: 0.2182, train/util_ratio: 0.5938, train/run_time: 0.1414, lr: 0.0005, train/prefecth_time: 0.0144 \n", + "1600 iteration USE_EMA: True, train/sup_loss: 0.0051, train/unsup_loss: 0.4610, train/total_loss: 0.4661, train/util_ratio: 0.6875, train/run_time: 0.1409, lr: 0.0005, train/prefecth_time: 0.0031 \n", + "1650 iteration USE_EMA: True, train/sup_loss: 0.0011, train/unsup_loss: 0.2337, train/total_loss: 0.2348, train/util_ratio: 0.8438, train/run_time: 0.1416, lr: 0.0004, train/prefecth_time: 0.0029 \n", + "1700 iteration USE_EMA: True, train/sup_loss: 0.0014, train/unsup_loss: 0.1703, train/total_loss: 0.1717, train/util_ratio: 0.6562, train/run_time: 0.1406, lr: 0.0004, train/prefecth_time: 0.0035 \n", + "1750 iteration USE_EMA: True, train/sup_loss: 0.2324, train/unsup_loss: 0.1236, train/total_loss: 0.3560, train/util_ratio: 0.5938, train/run_time: 0.1411, lr: 0.0004, train/prefecth_time: 0.0034 \n", + "1800 iteration USE_EMA: True, train/sup_loss: 0.0040, train/unsup_loss: 0.1355, train/total_loss: 0.1395, train/util_ratio: 0.8125, train/run_time: 0.1422, lr: 0.0004, train/prefecth_time: 0.0043 \n", + "1850 iteration USE_EMA: True, train/sup_loss: 0.0004, train/unsup_loss: 0.2151, train/total_loss: 0.2155, train/util_ratio: 0.6875, train/run_time: 0.1405, lr: 0.0004, train/prefecth_time: 0.0032 \n", + "1900 iteration USE_EMA: True, train/sup_loss: 0.0412, train/unsup_loss: 0.0414, train/total_loss: 0.0826, train/util_ratio: 0.6562, train/run_time: 0.1419, lr: 0.0004, train/prefecth_time: 0.0061 \n", + "1950 iteration USE_EMA: True, train/sup_loss: 0.0033, train/unsup_loss: 0.2867, train/total_loss: 0.2899, train/util_ratio: 0.7188, train/run_time: 0.1408, lr: 0.0004, train/prefecth_time: 0.0026 \n", + "validating...\n", + "confusion matrix:\n", + "[[0.966 0.002 0.001 0.002 0. 0.001 0.002 0.001 0.008 0.017]\n", + " [0. 0.965 0. 0. 0. 0. 0. 0. 0. 0.035]\n", + " [0.055 0. 0.707 0.026 0.044 0.067 0.016 0.079 0. 0.006]\n", + " [0.005 0.001 0.006 0.759 0.005 0.163 0.021 0.037 0.002 0.001]\n", + " [0.006 0. 0.003 0.018 0.874 0.01 0.004 0.084 0.001 0. ]\n", + " [0. 0. 0.001 0.077 0.006 0.856 0.001 0.059 0. 0. ]\n", + " [0.004 0. 0.003 0.001 0. 0.002 0.987 0.003 0. 0. ]\n", + " [0.006 0. 0.001 0.009 0.07 0.101 0. 0.81 0. 0.003]\n", + " [0.076 0.014 0.001 0.004 0. 0. 0. 0.001 0.836 0.068]\n", + " [0.007 0.015 0. 0. 0. 0. 0. 0. 0.002 0.976]]\n", + "model saved: ./saved_models/fixmatch/latest_model.pth\n", + "model saved: ./saved_models/fixmatch/model_best.pth\n", + "2000 iteration, USE_EMA: True, train/sup_loss: 0.0084, train/unsup_loss: 0.6485, train/total_loss: 0.6569, train/util_ratio: 0.7812, train/run_time: 0.1408, eval/loss: 0.3931, eval/top-1-acc: 0.8736, eval/balanced_acc: 0.8736, eval/precision: 0.8818, eval/recall: 0.8736, eval/F1: 0.8734, lr: 0.0004, train/prefecth_time: 0.0031 BEST_EVAL_ACC: 0.8736, at 2000 iters\n", + "2050 iteration USE_EMA: True, train/sup_loss: 0.0020, train/unsup_loss: 0.1562, train/total_loss: 0.1582, train/util_ratio: 0.7500, train/run_time: 0.1408, lr: 0.0004, train/prefecth_time: 0.0034 \n", + "2100 iteration USE_EMA: True, train/sup_loss: 0.0022, train/unsup_loss: 0.3097, train/total_loss: 0.3120, train/util_ratio: 0.8438, train/run_time: 0.1409, lr: 0.0004, train/prefecth_time: 0.0032 \n", + "2150 iteration USE_EMA: True, train/sup_loss: 0.0062, train/unsup_loss: 0.1458, train/total_loss: 0.1520, train/util_ratio: 0.7188, train/run_time: 0.1410, lr: 0.0004, train/prefecth_time: 0.0033 \n", + "2200 iteration USE_EMA: True, train/sup_loss: 0.0148, train/unsup_loss: 0.2113, train/total_loss: 0.2261, train/util_ratio: 0.6875, train/run_time: 0.1407, lr: 0.0004, train/prefecth_time: 0.0034 \n", + "2250 iteration USE_EMA: True, train/sup_loss: 0.0108, train/unsup_loss: 0.2698, train/total_loss: 0.2806, train/util_ratio: 0.6250, train/run_time: 0.1403, lr: 0.0004, train/prefecth_time: 0.0031 \n", + "2300 iteration USE_EMA: True, train/sup_loss: 0.0005, train/unsup_loss: 0.1321, train/total_loss: 0.1326, train/util_ratio: 0.8125, train/run_time: 0.1406, lr: 0.0004, train/prefecth_time: 0.0032 \n", + "2350 iteration USE_EMA: True, train/sup_loss: 0.0099, train/unsup_loss: 0.3188, train/total_loss: 0.3287, train/util_ratio: 0.7812, train/run_time: 0.1407, lr: 0.0004, train/prefecth_time: 0.0032 \n", + "2400 iteration USE_EMA: True, train/sup_loss: 0.0007, train/unsup_loss: 0.0553, train/total_loss: 0.0560, train/util_ratio: 0.7188, train/run_time: 0.1409, lr: 0.0004, train/prefecth_time: 0.0033 \n", + "2450 iteration USE_EMA: True, train/sup_loss: 0.0005, train/unsup_loss: 0.3062, train/total_loss: 0.3067, train/util_ratio: 0.8125, train/run_time: 0.1406, lr: 0.0004, train/prefecth_time: 0.0032 \n", + "validating...\n", + "confusion matrix:\n", + "[[0.968 0.002 0.001 0.001 0. 0. 0.002 0.001 0.013 0.012]\n", + " [0. 0.979 0. 0. 0. 0. 0. 0. 0. 0.021]\n", + " [0.061 0. 0.72 0.028 0.084 0.043 0.019 0.042 0. 0.003]\n", + " [0.005 0.001 0.003 0.805 0.01 0.125 0.025 0.023 0.002 0.001]\n", + " [0.004 0. 0.002 0.009 0.926 0.002 0.004 0.052 0.001 0. ]\n", + " [0. 0. 0.001 0.077 0.01 0.866 0.001 0.045 0. 0. ]\n", + " [0.004 0. 0.002 0. 0. 0.002 0.991 0.001 0. 0. ]\n", + " [0.008 0. 0.001 0.007 0.124 0.077 0. 0.782 0. 0.001]\n", + " [0.036 0.008 0. 0. 0. 0. 0. 0. 0.931 0.025]\n", + " [0.007 0.018 0. 0. 0. 0. 0. 0. 0.002 0.973]]\n", + "model saved: ./saved_models/fixmatch/latest_model.pth\n", + "model saved: ./saved_models/fixmatch/model_best.pth\n", + "2500 iteration, USE_EMA: True, train/sup_loss: 0.0004, train/unsup_loss: 0.0543, train/total_loss: 0.0547, train/util_ratio: 0.5938, train/run_time: 0.1402, eval/loss: 0.3548, eval/top-1-acc: 0.8941, eval/balanced_acc: 0.8941, eval/precision: 0.8989, eval/recall: 0.8941, eval/F1: 0.8932, lr: 0.0004, train/prefecth_time: 0.0033 BEST_EVAL_ACC: 0.8941, at 2500 iters\n", + "2550 iteration USE_EMA: True, train/sup_loss: 0.0035, train/unsup_loss: 0.1280, train/total_loss: 0.1314, train/util_ratio: 0.7812, train/run_time: 0.1404, lr: 0.0004, train/prefecth_time: 0.0031 \n", + "2600 iteration USE_EMA: True, train/sup_loss: 0.0007, train/unsup_loss: 0.2929, train/total_loss: 0.2936, train/util_ratio: 0.8125, train/run_time: 0.1402, lr: 0.0004, train/prefecth_time: 0.0032 \n", + "2650 iteration USE_EMA: True, train/sup_loss: 0.0002, train/unsup_loss: 0.0830, train/total_loss: 0.0832, train/util_ratio: 0.6562, train/run_time: 0.1401, lr: 0.0004, train/prefecth_time: 0.0034 \n", + "2700 iteration USE_EMA: True, train/sup_loss: 0.0005, train/unsup_loss: 0.0310, train/total_loss: 0.0315, train/util_ratio: 0.7812, train/run_time: 0.1405, lr: 0.0004, train/prefecth_time: 0.0031 \n", + "2750 iteration USE_EMA: True, train/sup_loss: 0.0004, train/unsup_loss: 0.2103, train/total_loss: 0.2107, train/util_ratio: 0.6562, train/run_time: 0.1396, lr: 0.0004, train/prefecth_time: 0.0034 \n", + "2800 iteration USE_EMA: True, train/sup_loss: 0.0028, train/unsup_loss: 0.1549, train/total_loss: 0.1577, train/util_ratio: 0.7812, train/run_time: 0.1401, lr: 0.0004, train/prefecth_time: 0.0032 \n", + "2850 iteration USE_EMA: True, train/sup_loss: 0.0170, train/unsup_loss: 0.0704, train/total_loss: 0.0874, train/util_ratio: 0.7500, train/run_time: 0.1424, lr: 0.0004, train/prefecth_time: 0.0032 \n", + "2900 iteration USE_EMA: True, train/sup_loss: 0.0367, train/unsup_loss: 0.0295, train/total_loss: 0.0661, train/util_ratio: 0.7188, train/run_time: 0.1401, lr: 0.0003, train/prefecth_time: 0.0029 \n", + "2950 iteration USE_EMA: True, train/sup_loss: 0.0009, train/unsup_loss: 0.2047, train/total_loss: 0.2056, train/util_ratio: 0.8438, train/run_time: 0.1405, lr: 0.0003, train/prefecth_time: 0.0031 \n", + "validating...\n", + "confusion matrix:\n", + "[[0.968 0.002 0.001 0.001 0. 0. 0.001 0.001 0.016 0.01 ]\n", + " [0. 0.984 0. 0. 0. 0. 0. 0. 0. 0.016]\n", + " [0.065 0. 0.722 0.022 0.105 0.036 0.02 0.026 0. 0.004]\n", + " [0.004 0.001 0.003 0.829 0.009 0.106 0.024 0.02 0.003 0.001]\n", + " [0.003 0. 0. 0.005 0.946 0.001 0.005 0.039 0.001 0. ]\n", + " [0. 0. 0.001 0.08 0.01 0.87 0.002 0.037 0. 0. ]\n", + " [0.004 0. 0.002 0. 0. 0.002 0.992 0. 0. 0. ]\n", + " [0.009 0. 0.001 0.006 0.138 0.064 0. 0.781 0. 0.001]\n", + " [0.023 0.008 0. 0. 0. 0. 0. 0. 0.953 0.016]\n", + " [0.008 0.022 0. 0. 0. 0. 0. 0. 0.002 0.968]]\n", + "model saved: ./saved_models/fixmatch/latest_model.pth\n", + "model saved: ./saved_models/fixmatch/model_best.pth\n", + "3000 iteration, USE_EMA: True, train/sup_loss: 0.0026, train/unsup_loss: 0.1566, train/total_loss: 0.1592, train/util_ratio: 0.7812, train/run_time: 0.1406, eval/loss: 0.3531, eval/top-1-acc: 0.9013, eval/balanced_acc: 0.9013, eval/precision: 0.9062, eval/recall: 0.9013, eval/F1: 0.9003, lr: 0.0003, train/prefecth_time: 0.0031 BEST_EVAL_ACC: 0.9013, at 3000 iters\n", + "3050 iteration USE_EMA: True, train/sup_loss: 0.0004, train/unsup_loss: 0.3025, train/total_loss: 0.3029, train/util_ratio: 0.7500, train/run_time: 0.1398, lr: 0.0003, train/prefecth_time: 0.0033 \n", + "3100 iteration USE_EMA: True, train/sup_loss: 0.0382, train/unsup_loss: 0.1407, train/total_loss: 0.1789, train/util_ratio: 0.6562, train/run_time: 0.1406, lr: 0.0003, train/prefecth_time: 0.0033 \n", + "3150 iteration USE_EMA: True, train/sup_loss: 0.0020, train/unsup_loss: 0.0475, train/total_loss: 0.0495, train/util_ratio: 0.6875, train/run_time: 0.1407, lr: 0.0003, train/prefecth_time: 0.0033 \n", + "3200 iteration USE_EMA: True, train/sup_loss: 0.0009, train/unsup_loss: 0.1359, train/total_loss: 0.1368, train/util_ratio: 0.6875, train/run_time: 0.1404, lr: 0.0003, train/prefecth_time: 0.0035 \n", + "3250 iteration USE_EMA: True, train/sup_loss: 0.0066, train/unsup_loss: 0.1037, train/total_loss: 0.1103, train/util_ratio: 0.7812, train/run_time: 0.1404, lr: 0.0003, train/prefecth_time: 0.0033 \n", + "3300 iteration USE_EMA: True, train/sup_loss: 0.0005, train/unsup_loss: 0.0957, train/total_loss: 0.0963, train/util_ratio: 0.6562, train/run_time: 0.1397, lr: 0.0003, train/prefecth_time: 0.0031 \n", + "3350 iteration USE_EMA: True, train/sup_loss: 0.0005, train/unsup_loss: 0.0205, train/total_loss: 0.0210, train/util_ratio: 0.6562, train/run_time: 0.1402, lr: 0.0003, train/prefecth_time: 0.0033 \n", + "3400 iteration USE_EMA: True, train/sup_loss: 0.1098, train/unsup_loss: 0.0330, train/total_loss: 0.1428, train/util_ratio: 0.6875, train/run_time: 0.1400, lr: 0.0003, train/prefecth_time: 0.0059 \n", + "3450 iteration USE_EMA: True, train/sup_loss: 0.0017, train/unsup_loss: 0.1417, train/total_loss: 0.1435, train/util_ratio: 0.6875, train/run_time: 0.1410, lr: 0.0003, train/prefecth_time: 0.0031 \n", + "validating...\n", + "confusion matrix:\n", + "[[0.968 0.002 0.001 0.001 0. 0. 0.001 0.001 0.017 0.009]\n", + " [0. 0.986 0. 0. 0. 0. 0. 0. 0. 0.014]\n", + " [0.063 0. 0.728 0.02 0.111 0.035 0.02 0.021 0. 0.002]\n", + " [0.004 0.001 0.002 0.853 0.007 0.094 0.021 0.015 0.002 0.001]\n", + " [0.003 0. 0. 0.004 0.946 0.001 0.006 0.039 0.001 0. ]\n", + " [0. 0. 0.001 0.067 0.009 0.888 0.002 0.033 0. 0. ]\n", + " [0.004 0. 0.001 0. 0. 0.002 0.993 0. 0. 0. ]\n", + " [0.008 0. 0.002 0.007 0.132 0.064 0.001 0.785 0. 0.001]\n", + " [0.021 0.007 0. 0. 0. 0. 0. 0. 0.959 0.013]\n", + " [0.006 0.023 0. 0. 0. 0. 0. 0. 0.002 0.969]]\n", + "model saved: ./saved_models/fixmatch/latest_model.pth\n", + "model saved: ./saved_models/fixmatch/model_best.pth\n", + "3500 iteration, USE_EMA: True, train/sup_loss: 0.0006, train/unsup_loss: 0.0635, train/total_loss: 0.0642, train/util_ratio: 0.7188, train/run_time: 0.1406, eval/loss: 0.3464, eval/top-1-acc: 0.9075, eval/balanced_acc: 0.9075, eval/precision: 0.9124, eval/recall: 0.9075, eval/F1: 0.9066, lr: 0.0003, train/prefecth_time: 0.0033 BEST_EVAL_ACC: 0.9075, at 3500 iters\n", + "3550 iteration USE_EMA: True, train/sup_loss: 0.0207, train/unsup_loss: 0.1370, train/total_loss: 0.1577, train/util_ratio: 0.7188, train/run_time: 0.1400, lr: 0.0003, train/prefecth_time: 0.0034 \n", + "3600 iteration USE_EMA: True, train/sup_loss: 0.0003, train/unsup_loss: 0.2376, train/total_loss: 0.2379, train/util_ratio: 0.8125, train/run_time: 0.1406, lr: 0.0003, train/prefecth_time: 0.0034 \n", + "3650 iteration USE_EMA: True, train/sup_loss: 0.0203, train/unsup_loss: 0.0581, train/total_loss: 0.0784, train/util_ratio: 0.6875, train/run_time: 0.1405, lr: 0.0003, train/prefecth_time: 0.0034 \n", + "3700 iteration USE_EMA: True, train/sup_loss: 0.0291, train/unsup_loss: 0.1788, train/total_loss: 0.2078, train/util_ratio: 0.8438, train/run_time: 0.1406, lr: 0.0003, train/prefecth_time: 0.0032 \n", + "3750 iteration USE_EMA: True, train/sup_loss: 0.0020, train/unsup_loss: 0.0720, train/total_loss: 0.0739, train/util_ratio: 0.7500, train/run_time: 0.1400, lr: 0.0003, train/prefecth_time: 0.0034 \n", + "3800 iteration USE_EMA: True, train/sup_loss: 0.0145, train/unsup_loss: 0.1374, train/total_loss: 0.1519, train/util_ratio: 0.8750, train/run_time: 0.1407, lr: 0.0003, train/prefecth_time: 0.0033 \n", + "3850 iteration USE_EMA: True, train/sup_loss: 0.0262, train/unsup_loss: 0.0992, train/total_loss: 0.1254, train/util_ratio: 0.7188, train/run_time: 0.1410, lr: 0.0002, train/prefecth_time: 0.0030 \n", + "3900 iteration USE_EMA: True, train/sup_loss: 0.0003, train/unsup_loss: 0.3316, train/total_loss: 0.3319, train/util_ratio: 0.7812, train/run_time: 0.1407, lr: 0.0002, train/prefecth_time: 0.0033 \n", + "3950 iteration USE_EMA: True, train/sup_loss: 0.0112, train/unsup_loss: 0.1994, train/total_loss: 0.2106, train/util_ratio: 0.7188, train/run_time: 0.1408, lr: 0.0002, train/prefecth_time: 0.0031 \n", + "validating...\n", + "confusion matrix:\n", + "[[0.968 0.001 0.001 0.001 0. 0. 0.001 0.001 0.018 0.009]\n", + " [0. 0.988 0. 0. 0. 0. 0. 0. 0. 0.012]\n", + " [0.057 0. 0.749 0.015 0.118 0.023 0.021 0.015 0. 0.002]\n", + " [0.005 0.001 0.001 0.87 0.008 0.077 0.025 0.01 0.002 0.001]\n", + " [0.001 0. 0. 0.005 0.955 0. 0.008 0.03 0.001 0. ]\n", + " [0. 0. 0.001 0.068 0.01 0.887 0.003 0.031 0. 0. ]\n", + " [0.003 0. 0.001 0.001 0. 0.001 0.993 0. 0.001 0. ]\n", + " [0.008 0. 0.002 0.009 0.132 0.058 0.001 0.789 0. 0.001]\n", + " [0.017 0.007 0. 0. 0. 0. 0. 0. 0.967 0.009]\n", + " [0.006 0.025 0. 0. 0. 0. 0. 0. 0.002 0.967]]\n", + "model saved: ./saved_models/fixmatch/latest_model.pth\n", + "model saved: ./saved_models/fixmatch/model_best.pth\n", + "4000 iteration, USE_EMA: True, train/sup_loss: 0.0005, train/unsup_loss: 0.5238, train/total_loss: 0.5243, train/util_ratio: 0.9062, train/run_time: 0.1405, eval/loss: 0.3374, eval/top-1-acc: 0.9133, eval/balanced_acc: 0.9133, eval/precision: 0.9181, eval/recall: 0.9133, eval/F1: 0.9125, lr: 0.0002, train/prefecth_time: 0.0032 BEST_EVAL_ACC: 0.9133, at 4000 iters\n", + "4050 iteration USE_EMA: True, train/sup_loss: 0.0208, train/unsup_loss: 0.3388, train/total_loss: 0.3596, train/util_ratio: 0.6250, train/run_time: 0.1396, lr: 0.0002, train/prefecth_time: 0.0031 \n", + "4100 iteration USE_EMA: True, train/sup_loss: 0.3715, train/unsup_loss: 0.0317, train/total_loss: 0.4031, train/util_ratio: 0.8438, train/run_time: 0.1403, lr: 0.0002, train/prefecth_time: 0.0034 \n", + "4150 iteration USE_EMA: True, train/sup_loss: 0.0386, train/unsup_loss: 0.3697, train/total_loss: 0.4083, train/util_ratio: 0.6875, train/run_time: 0.1405, lr: 0.0002, train/prefecth_time: 0.0035 \n", + "4200 iteration USE_EMA: True, train/sup_loss: 0.0003, train/unsup_loss: 0.1447, train/total_loss: 0.1449, train/util_ratio: 0.8750, train/run_time: 0.1401, lr: 0.0002, train/prefecth_time: 0.0032 \n", + "4250 iteration USE_EMA: True, train/sup_loss: 0.0015, train/unsup_loss: 0.2502, train/total_loss: 0.2517, train/util_ratio: 0.8438, train/run_time: 0.1406, lr: 0.0002, train/prefecth_time: 0.0031 \n", + "4300 iteration USE_EMA: True, train/sup_loss: 0.0016, train/unsup_loss: 0.4713, train/total_loss: 0.4729, train/util_ratio: 0.6562, train/run_time: 0.1403, lr: 0.0002, train/prefecth_time: 0.0033 \n", + "4350 iteration USE_EMA: True, train/sup_loss: 0.0020, train/unsup_loss: 0.2388, train/total_loss: 0.2408, train/util_ratio: 0.7500, train/run_time: 0.1401, lr: 0.0002, train/prefecth_time: 0.0031 \n", + "4400 iteration USE_EMA: True, train/sup_loss: 0.0007, train/unsup_loss: 0.1646, train/total_loss: 0.1653, train/util_ratio: 0.9062, train/run_time: 0.1406, lr: 0.0002, train/prefecth_time: 0.0031 \n", + "4450 iteration USE_EMA: True, train/sup_loss: 0.0006, train/unsup_loss: 0.0542, train/total_loss: 0.0549, train/util_ratio: 0.8438, train/run_time: 0.1408, lr: 0.0002, train/prefecth_time: 0.0031 \n", + "validating...\n", + "confusion matrix:\n", + "[[0.967 0.001 0.001 0.001 0. 0. 0.001 0.001 0.019 0.009]\n", + " [0. 0.988 0. 0. 0. 0. 0. 0. 0. 0.012]\n", + " [0.052 0. 0.761 0.013 0.119 0.021 0.018 0.014 0. 0.002]\n", + " [0.005 0.001 0.001 0.881 0.008 0.068 0.025 0.008 0.002 0.001]\n", + " [0. 0. 0.001 0.005 0.962 0. 0.008 0.023 0.001 0. ]\n", + " [0. 0. 0.001 0.068 0.01 0.886 0.004 0.031 0. 0. ]\n", + " [0.003 0. 0.001 0.002 0. 0.001 0.992 0. 0.001 0. ]\n", + " [0.007 0. 0.002 0.01 0.127 0.057 0. 0.795 0. 0.002]\n", + " [0.014 0.007 0. 0. 0. 0. 0. 0. 0.97 0.009]\n", + " [0.005 0.025 0. 0. 0. 0. 0. 0. 0.002 0.968]]\n", + "model saved: ./saved_models/fixmatch/latest_model.pth\n", + "model saved: ./saved_models/fixmatch/model_best.pth\n", + "4500 iteration, USE_EMA: True, train/sup_loss: 0.0003, train/unsup_loss: 0.2390, train/total_loss: 0.2393, train/util_ratio: 0.7188, train/run_time: 0.1398, eval/loss: 0.3292, eval/top-1-acc: 0.9170, eval/balanced_acc: 0.9170, eval/precision: 0.9216, eval/recall: 0.9170, eval/F1: 0.9163, lr: 0.0002, train/prefecth_time: 0.0034 BEST_EVAL_ACC: 0.9170, at 4500 iters\n", + "4550 iteration USE_EMA: True, train/sup_loss: 0.0047, train/unsup_loss: 0.1983, train/total_loss: 0.2030, train/util_ratio: 0.7812, train/run_time: 0.1402, lr: 0.0002, train/prefecth_time: 0.0034 \n", + "4600 iteration USE_EMA: True, train/sup_loss: 0.1438, train/unsup_loss: 0.0164, train/total_loss: 0.1602, train/util_ratio: 0.6250, train/run_time: 0.1408, lr: 0.0002, train/prefecth_time: 0.0034 \n", + "4650 iteration USE_EMA: True, train/sup_loss: 0.0061, train/unsup_loss: 0.1227, train/total_loss: 0.1288, train/util_ratio: 0.7188, train/run_time: 0.1407, lr: 0.0001, train/prefecth_time: 0.0033 \n", + "4700 iteration USE_EMA: True, train/sup_loss: 0.0008, train/unsup_loss: 0.2121, train/total_loss: 0.2130, train/util_ratio: 0.7188, train/run_time: 0.1403, lr: 0.0001, train/prefecth_time: 0.0033 \n", + "4750 iteration USE_EMA: True, train/sup_loss: 0.0004, train/unsup_loss: 0.3475, train/total_loss: 0.3480, train/util_ratio: 0.8125, train/run_time: 0.1406, lr: 0.0001, train/prefecth_time: 0.0031 \n", + "4800 iteration USE_EMA: True, train/sup_loss: 0.0002, train/unsup_loss: 0.1207, train/total_loss: 0.1208, train/util_ratio: 0.7812, train/run_time: 0.1405, lr: 0.0001, train/prefecth_time: 0.0034 \n", + "4850 iteration USE_EMA: True, train/sup_loss: 0.0026, train/unsup_loss: 0.1368, train/total_loss: 0.1394, train/util_ratio: 0.7812, train/run_time: 0.1402, lr: 0.0001, train/prefecth_time: 0.0032 \n", + "4900 iteration USE_EMA: True, train/sup_loss: 0.0019, train/unsup_loss: 0.0960, train/total_loss: 0.0979, train/util_ratio: 0.5938, train/run_time: 0.1401, lr: 0.0001, train/prefecth_time: 0.0031 \n", + "4950 iteration USE_EMA: True, train/sup_loss: 0.0007, train/unsup_loss: 0.0922, train/total_loss: 0.0929, train/util_ratio: 0.7500, train/run_time: 0.1403, lr: 0.0001, train/prefecth_time: 0.0032 \n", + "validating...\n", + "confusion matrix:\n", + "[[0.968 0.001 0. 0.001 0. 0. 0.001 0.001 0.019 0.009]\n", + " [0. 0.988 0. 0. 0. 0. 0. 0. 0. 0.012]\n", + " [0.05 0. 0.774 0.008 0.119 0.02 0.018 0.01 0. 0.001]\n", + " [0.004 0.001 0.003 0.874 0.011 0.07 0.026 0.008 0.002 0.001]\n", + " [0. 0. 0.001 0.003 0.965 0.001 0.008 0.021 0.001 0. ]\n", + " [0. 0. 0.001 0.061 0.01 0.895 0.004 0.029 0. 0. ]\n", + " [0.003 0. 0.001 0.001 0.001 0.001 0.992 0. 0.001 0. ]\n", + " [0.007 0. 0.002 0.009 0.126 0.058 0. 0.796 0. 0.002]\n", + " [0.014 0.007 0. 0. 0. 0. 0. 0. 0.972 0.007]\n", + " [0.005 0.023 0. 0. 0. 0. 0. 0. 0.002 0.97 ]]\n", + "model saved: ./saved_models/fixmatch/latest_model.pth\n", + "model saved: ./saved_models/fixmatch/model_best.pth\n", + "5000 iteration, USE_EMA: True, train/sup_loss: 0.0012, train/unsup_loss: 0.1903, train/total_loss: 0.1915, train/util_ratio: 0.7188, train/run_time: 0.1399, eval/loss: 0.3270, eval/top-1-acc: 0.9194, eval/balanced_acc: 0.9194, eval/precision: 0.9241, eval/recall: 0.9194, eval/F1: 0.9188, lr: 0.0001, train/prefecth_time: 0.0028 BEST_EVAL_ACC: 0.9194, at 5000 iters\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "/home/haoc/miniconda3/envs/test/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "[2022-12-18 16:02:52,476 INFO] confusion matrix\n", - "[2022-12-18 16:02:52,478 INFO] [[0. 0. 0. 1. 0. 0. 0. 0. 0. 0.]\n", - " [0. 0. 0. 1. 0. 0. 0. 0. 0. 0.]\n", - " [0. 0. 0. 1. 0. 0. 0. 0. 0. 0.]\n", - " [0. 0. 0. 1. 0. 0. 0. 0. 0. 0.]\n", - " [0. 0. 0. 1. 0. 0. 0. 0. 0. 0.]\n", - " [0. 0. 0. 1. 0. 0. 0. 0. 0. 0.]\n", - " [0. 0. 0. 1. 0. 0. 0. 0. 0. 0.]\n", - " [0. 0. 0. 1. 0. 0. 0. 0. 0. 0.]\n", - " [0. 0. 0. 1. 0. 0. 0. 0. 0. 0.]\n", - " [0. 0. 0. 1. 0. 0. 0. 0. 0. 0.]]\n", - "[2022-12-18 16:02:52,481 INFO] evaluation metric\n", - "[2022-12-18 16:02:52,483 INFO] acc: 0.1000\n", - "[2022-12-18 16:02:52,484 INFO] precision: 0.0100\n", - "[2022-12-18 16:02:52,485 INFO] recall: 0.1000\n", - "[2022-12-18 16:02:52,486 INFO] f1: 0.0182\n" + "[2023-07-19 23:45:48,260 INFO] confusion matrix\n", + "[2023-07-19 23:45:48,261 INFO] [[0.969 0.001 0. 0.001 0. 0. 0.001 0.001 0.018 0.009]\n", + " [0. 0.987 0. 0. 0. 0. 0. 0. 0. 0.013]\n", + " [0.049 0. 0.776 0.007 0.127 0.017 0.016 0.007 0. 0.001]\n", + " [0.003 0.001 0.006 0.869 0.013 0.072 0.027 0.006 0.002 0.001]\n", + " [0. 0. 0. 0.002 0.974 0.001 0.004 0.018 0.001 0. ]\n", + " [0. 0. 0.001 0.045 0.011 0.916 0.003 0.024 0. 0. ]\n", + " [0.004 0. 0.001 0.001 0.001 0.001 0.991 0. 0.001 0. ]\n", + " [0.006 0. 0.002 0.006 0.127 0.059 0. 0.798 0. 0.002]\n", + " [0.014 0.007 0. 0. 0. 0. 0. 0. 0.973 0.006]\n", + " [0.005 0.018 0. 0. 0. 0. 0. 0. 0.002 0.975]]\n", + "[2023-07-19 23:45:48,264 INFO] evaluation metric\n", + "[2023-07-19 23:45:48,264 INFO] acc: 0.9228\n", + "[2023-07-19 23:45:48,265 INFO] precision: 0.9283\n", + "[2023-07-19 23:45:48,266 INFO] recall: 0.9228\n", + "[2023-07-19 23:45:48,266 INFO] f1: 0.9223\n" ] }, { @@ -316,8 +584,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2022-12-18 16:02:53,467 INFO] Best acc 0.1000 at epoch 0\n", - "[2022-12-18 16:02:53,468 INFO] Training finished.\n" + "[2023-07-19 23:45:50,377 INFO] Best acc 0.9228 at epoch 0\n", + "[2023-07-19 23:45:50,378 INFO] Training finished.\n" ] }, { @@ -331,30 +599,31 @@ "name": "stderr", "output_type": "stream", "text": [ - "/home/haoc/miniconda3/envs/test/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "[2022-12-18 16:04:06,346 INFO] confusion matrix\n", - "[2022-12-18 16:04:06,348 INFO] [[0. 0. 0. 1. 0. 0. 0. 0. 0. 0.]\n", - " [0. 0. 0. 1. 0. 0. 0. 0. 0. 0.]\n", - " [0. 0. 0. 1. 0. 0. 0. 0. 0. 0.]\n", - " [0. 0. 0. 1. 0. 0. 0. 0. 0. 0.]\n", - " [0. 0. 0. 1. 0. 0. 0. 0. 0. 0.]\n", - " [0. 0. 0. 1. 0. 0. 0. 0. 0. 0.]\n", - " [0. 0. 0. 1. 0. 0. 0. 0. 0. 0.]\n", - " [0. 0. 0. 1. 0. 0. 0. 0. 0. 0.]\n", - " [0. 0. 0. 1. 0. 0. 0. 0. 0. 0.]\n", - " [0. 0. 0. 1. 0. 0. 0. 0. 0. 0.]]\n", - "[2022-12-18 16:04:06,351 INFO] evaluation metric\n", - "[2022-12-18 16:04:06,352 INFO] acc: 0.1000\n", - "[2022-12-18 16:04:06,353 INFO] precision: 0.0100\n", - "[2022-12-18 16:04:06,354 INFO] recall: 0.1000\n", - "[2022-12-18 16:04:06,354 INFO] f1: 0.0182\n" + "[2023-07-19 23:46:52,956 INFO] confusion matrix\n", + "[2023-07-19 23:46:52,958 INFO] [[0.969 0.001 0. 0.001 0. 0. 0.001 0.001 0.018 0.009]\n", + " [0. 0.987 0. 0. 0. 0. 0. 0. 0. 0.013]\n", + " [0.049 0. 0.776 0.007 0.127 0.017 0.016 0.007 0. 0.001]\n", + " [0.003 0.001 0.006 0.869 0.013 0.072 0.027 0.006 0.002 0.001]\n", + " [0. 0. 0. 0.002 0.974 0.001 0.004 0.018 0.001 0. ]\n", + " [0. 0. 0.001 0.045 0.011 0.916 0.003 0.024 0. 0. ]\n", + " [0.004 0. 0.001 0.001 0.001 0.001 0.991 0. 0.001 0. ]\n", + " [0.006 0. 0.002 0.006 0.127 0.059 0. 0.798 0. 0.002]\n", + " [0.014 0.007 0. 0. 0. 0. 0. 0. 0.973 0.006]\n", + " [0.005 0.018 0. 0. 0. 0. 0. 0. 0.002 0.975]]\n", + "[2023-07-19 23:46:52,961 INFO] evaluation metric\n", + "[2023-07-19 23:46:52,961 INFO] acc: 0.9228\n", + "[2023-07-19 23:46:52,962 INFO] precision: 0.9283\n", + "[2023-07-19 23:46:52,963 INFO] recall: 0.9228\n", + "[2023-07-19 23:46:52,963 INFO] f1: 0.9223\n" ] }, { "data": { "text/plain": [ - "{'acc': 0.1, 'precision': 0.01, 'recall': 0.1, 'f1': 0.01818181818181818}" + "{'acc': 0.9228,\n", + " 'precision': 0.928316578833672,\n", + " 'recall': 0.9228,\n", + " 'f1': 0.9223419792678316}" ] }, "execution_count": 6, @@ -386,7 +655,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.13" + "version": "3.9.17" }, "vscode": { "interpreter": { diff --git a/notebooks/Custom_Dataset.ipynb b/notebooks/Custom_Dataset.ipynb index dd4b470af..7ab467cde 100644 --- a/notebooks/Custom_Dataset.ipynb +++ b/notebooks/Custom_Dataset.ipynb @@ -33,7 +33,16 @@ "name": "#%%\n" } }, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/media/Zeus/haoc/miniconda/envs/test_semilearn_031/lib/python3.9/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], "source": [ "import numpy as np\n", "from torchvision import transforms\n", @@ -55,7 +64,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "metadata": { "collapsed": false, "pycharm": { @@ -63,13 +72,6 @@ } }, "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "here 1\n" - ] - }, { "name": "stderr", "output_type": "stream", @@ -87,10 +89,10 @@ " 'pretrain_path': 'https://github.com/microsoft/Semi-supervised-learning/releases/download/v.0.0.0/vit_tiny_patch2_32_mlp_im_1k_32.pth',\n", "\n", " # optimization configs\n", - " 'epoch': 5, # set to 100\n", - " 'num_train_iter': 5000, # set to 102400\n", - " 'num_eval_iter': 1000, # set to 1024\n", - " 'num_log_iter': 50, # set to 256\n", + " 'epoch': 1, \n", + " 'num_train_iter': 1000, \n", + " 'num_eval_iter': 500, \n", + " 'num_log_iter': 50, \n", " 'optim': 'AdamW',\n", " 'lr': 5e-4,\n", " 'layer_decay': 0.5,\n", @@ -121,7 +123,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "metadata": { "collapsed": false, "pycharm": { @@ -157,7 +159,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "metadata": { "collapsed": false, "pycharm": { @@ -189,7 +191,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "metadata": { "collapsed": false, "pycharm": { @@ -212,7 +214,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "metadata": { "collapsed": false, "pycharm": { @@ -241,7 +243,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, "metadata": { "collapsed": false, "pycharm": { @@ -253,204 +255,27 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch: 0\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/haoc/miniconda3/envs/test/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "[2022-12-18 15:24:18,639 INFO] confusion matrix\n", - "[2022-12-18 15:24:18,641 INFO] [[0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]]\n", - "[2022-12-18 15:24:18,644 INFO] evaluation metric\n", - "[2022-12-18 15:24:18,644 INFO] acc: 0.1000\n", - "[2022-12-18 15:24:18,646 INFO] precision: 0.0100\n", - "[2022-12-18 15:24:18,647 INFO] recall: 0.1000\n", - "[2022-12-18 15:24:18,648 INFO] f1: 0.0182\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "model saved: ./saved_models/fixmatch/latest_model.pth\n", - "model saved: ./saved_models/fixmatch/model_best.pth\n", - "Epoch: 1\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/haoc/miniconda3/envs/test/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "[2022-12-18 15:25:34,196 INFO] confusion matrix\n", - "[2022-12-18 15:25:34,198 INFO] [[0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]]\n", - "[2022-12-18 15:25:34,201 INFO] evaluation metric\n", - "[2022-12-18 15:25:34,203 INFO] acc: 0.1000\n", - "[2022-12-18 15:25:34,204 INFO] precision: 0.0100\n", - "[2022-12-18 15:25:34,204 INFO] recall: 0.1000\n", - "[2022-12-18 15:25:34,205 INFO] f1: 0.0182\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "model saved: ./saved_models/fixmatch/latest_model.pth\n", - "Epoch: 2\n" + "Epoch: 0\n", + "50 iteration USE_EMA: True, train/sup_loss: 2.3674, train/unsup_loss: 0.0000, train/total_loss: 2.3674, train/util_ratio: 0.0000, train/run_time: 0.1369, lr: 0.0005, train/prefecth_time: 0.0040 \n", + "100 iteration USE_EMA: True, train/sup_loss: 2.3490, train/unsup_loss: 0.0000, train/total_loss: 2.3490, train/util_ratio: 0.0000, train/run_time: 0.1373, lr: 0.0005, train/prefecth_time: 0.0038 \n", + "150 iteration USE_EMA: True, train/sup_loss: 2.3455, train/unsup_loss: 0.0000, train/total_loss: 2.3455, train/util_ratio: 0.0000, train/run_time: 0.1377, lr: 0.0005, train/prefecth_time: 0.0037 \n" ] }, { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/haoc/miniconda3/envs/test/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "[2022-12-18 15:26:49,069 INFO] confusion matrix\n", - "[2022-12-18 15:26:49,071 INFO] [[0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]]\n", - "[2022-12-18 15:26:49,073 INFO] evaluation metric\n", - "[2022-12-18 15:26:49,073 INFO] acc: 0.1000\n", - "[2022-12-18 15:26:49,075 INFO] precision: 0.0100\n", - "[2022-12-18 15:26:49,075 INFO] recall: 0.1000\n", - "[2022-12-18 15:26:49,077 INFO] f1: 0.0182\n" + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[8], line 3\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[39m# training and evaluation\u001b[39;00m\n\u001b[1;32m 2\u001b[0m trainer \u001b[39m=\u001b[39m Trainer(config, algorithm)\n\u001b[0;32m----> 3\u001b[0m trainer\u001b[39m.\u001b[39;49mfit(train_lb_loader, train_ulb_loader, eval_loader)\n\u001b[1;32m 4\u001b[0m trainer\u001b[39m.\u001b[39mevaluate(eval_loader)\n", + "File \u001b[0;32m/media/Zeus/haoc/Semi-supervised-learning/semilearn/lighting/trainer.py:63\u001b[0m, in \u001b[0;36mTrainer.fit\u001b[0;34m(self, train_lb_loader, train_ulb_loader, eval_loader)\u001b[0m\n\u001b[1;32m 61\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39malgorithm\u001b[39m.\u001b[39mout_dict \u001b[39m=\u001b[39m out_dict\n\u001b[1;32m 62\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39malgorithm\u001b[39m.\u001b[39mlog_dict \u001b[39m=\u001b[39m log_dict\n\u001b[0;32m---> 63\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49malgorithm\u001b[39m.\u001b[39;49mcall_hook(\u001b[39m\"\u001b[39;49m\u001b[39mafter_train_step\u001b[39;49m\u001b[39m\"\u001b[39;49m)\n\u001b[1;32m 65\u001b[0m bar\u001b[39m.\u001b[39msuffix \u001b[39m=\u001b[39m (\u001b[39m\"\u001b[39m\u001b[39mIter: \u001b[39m\u001b[39m{batch:4}\u001b[39;00m\u001b[39m/\u001b[39m\u001b[39m{iter:4}\u001b[39;00m\u001b[39m.\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m.\u001b[39mformat(batch\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39malgorithm\u001b[39m.\u001b[39mit, \u001b[39miter\u001b[39m\u001b[39m=\u001b[39m\u001b[39mlen\u001b[39m(train_lb_loader)))\n\u001b[1;32m 66\u001b[0m bar\u001b[39m.\u001b[39mnext()\n", + "File \u001b[0;32m/media/Zeus/haoc/Semi-supervised-learning/semilearn/core/algorithmbase.py:490\u001b[0m, in \u001b[0;36mAlgorithmBase.call_hook\u001b[0;34m(self, fn_name, hook_name, *args, **kwargs)\u001b[0m\n\u001b[1;32m 488\u001b[0m \u001b[39mfor\u001b[39;00m hook \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mhooks_dict\u001b[39m.\u001b[39mvalues():\n\u001b[1;32m 489\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mhasattr\u001b[39m(hook, fn_name):\n\u001b[0;32m--> 490\u001b[0m \u001b[39mgetattr\u001b[39;49m(hook, fn_name)(\u001b[39mself\u001b[39;49m, \u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n", + "File \u001b[0;32m/media/Zeus/haoc/Semi-supervised-learning/semilearn/core/hooks/param_update.py:33\u001b[0m, in \u001b[0;36mParamUpdateHook.after_train_step\u001b[0;34m(self, algorithm)\u001b[0m\n\u001b[1;32m 31\u001b[0m algorithm\u001b[39m.\u001b[39mloss_scaler\u001b[39m.\u001b[39mupdate()\n\u001b[1;32m 32\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m---> 33\u001b[0m loss\u001b[39m.\u001b[39;49mbackward()\n\u001b[1;32m 34\u001b[0m \u001b[39mif\u001b[39;00m (algorithm\u001b[39m.\u001b[39mclip_grad \u001b[39m>\u001b[39m \u001b[39m0\u001b[39m):\n\u001b[1;32m 35\u001b[0m torch\u001b[39m.\u001b[39mnn\u001b[39m.\u001b[39mutils\u001b[39m.\u001b[39mclip_grad_norm_(algorithm\u001b[39m.\u001b[39mmodel\u001b[39m.\u001b[39mparameters(), algorithm\u001b[39m.\u001b[39mclip_grad)\n", + "File \u001b[0;32m/media/Zeus/haoc/miniconda/envs/test_semilearn_031/lib/python3.9/site-packages/torch/_tensor.py:487\u001b[0m, in \u001b[0;36mTensor.backward\u001b[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[1;32m 477\u001b[0m \u001b[39mif\u001b[39;00m has_torch_function_unary(\u001b[39mself\u001b[39m):\n\u001b[1;32m 478\u001b[0m \u001b[39mreturn\u001b[39;00m handle_torch_function(\n\u001b[1;32m 479\u001b[0m Tensor\u001b[39m.\u001b[39mbackward,\n\u001b[1;32m 480\u001b[0m (\u001b[39mself\u001b[39m,),\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 485\u001b[0m inputs\u001b[39m=\u001b[39minputs,\n\u001b[1;32m 486\u001b[0m )\n\u001b[0;32m--> 487\u001b[0m torch\u001b[39m.\u001b[39;49mautograd\u001b[39m.\u001b[39;49mbackward(\n\u001b[1;32m 488\u001b[0m \u001b[39mself\u001b[39;49m, gradient, retain_graph, create_graph, inputs\u001b[39m=\u001b[39;49minputs\n\u001b[1;32m 489\u001b[0m )\n", + "File \u001b[0;32m/media/Zeus/haoc/miniconda/envs/test_semilearn_031/lib/python3.9/site-packages/torch/autograd/__init__.py:200\u001b[0m, in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[1;32m 195\u001b[0m retain_graph \u001b[39m=\u001b[39m create_graph\n\u001b[1;32m 197\u001b[0m \u001b[39m# The reason we repeat same the comment below is that\u001b[39;00m\n\u001b[1;32m 198\u001b[0m \u001b[39m# some Python versions print out the first line of a multi-line function\u001b[39;00m\n\u001b[1;32m 199\u001b[0m \u001b[39m# calls in the traceback and some print out the last line\u001b[39;00m\n\u001b[0;32m--> 200\u001b[0m Variable\u001b[39m.\u001b[39;49m_execution_engine\u001b[39m.\u001b[39;49mrun_backward( \u001b[39m# Calls into the C++ engine to run the backward pass\u001b[39;49;00m\n\u001b[1;32m 201\u001b[0m tensors, grad_tensors_, retain_graph, create_graph, inputs,\n\u001b[1;32m 202\u001b[0m allow_unreachable\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m, accumulate_grad\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m)\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "model saved: ./saved_models/fixmatch/latest_model.pth\n", - "Epoch: 3\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/haoc/miniconda3/envs/test/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "[2022-12-18 15:28:04,448 INFO] confusion matrix\n", - "[2022-12-18 15:28:04,450 INFO] [[0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]]\n", - "[2022-12-18 15:28:04,453 INFO] evaluation metric\n", - "[2022-12-18 15:28:04,454 INFO] acc: 0.1000\n", - "[2022-12-18 15:28:04,455 INFO] precision: 0.0100\n", - "[2022-12-18 15:28:04,456 INFO] recall: 0.1000\n", - "[2022-12-18 15:28:04,457 INFO] f1: 0.0182\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "model saved: ./saved_models/fixmatch/latest_model.pth\n", - "Epoch: 4\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/haoc/miniconda3/envs/test/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "[2022-12-18 15:29:19,589 INFO] confusion matrix\n", - "[2022-12-18 15:29:19,591 INFO] [[0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]]\n", - "[2022-12-18 15:29:19,594 INFO] evaluation metric\n", - "[2022-12-18 15:29:19,595 INFO] acc: 0.1000\n", - "[2022-12-18 15:29:19,597 INFO] precision: 0.0100\n", - "[2022-12-18 15:29:19,598 INFO] recall: 0.1000\n", - "[2022-12-18 15:29:19,599 INFO] f1: 0.0182\n", - "[2022-12-18 15:29:20,143 INFO] Best acc 0.1000 at epoch 0\n", - "[2022-12-18 15:29:20,145 INFO] Training finished.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "model saved: ./saved_models/fixmatch/latest_model.pth\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/haoc/miniconda3/envs/test/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n", - " _warn_prf(average, modifier, msg_start, len(result))\n", - "[2022-12-18 15:29:35,164 INFO] confusion matrix\n", - "[2022-12-18 15:29:35,166 INFO] [[0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n", - " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]]\n", - "[2022-12-18 15:29:35,169 INFO] evaluation metric\n", - "[2022-12-18 15:29:35,170 INFO] acc: 0.1000\n", - "[2022-12-18 15:29:35,171 INFO] precision: 0.0100\n", - "[2022-12-18 15:29:35,172 INFO] recall: 0.1000\n", - "[2022-12-18 15:29:35,173 INFO] f1: 0.0182\n" - ] - }, - { - "data": { - "text/plain": [ - "{'acc': 0.1, 'precision': 0.01, 'recall': 0.1, 'f1': 0.01818181818181818}" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" } ], "source": [ @@ -477,7 +302,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.13 (default, Mar 28 2022, 11:38:47) \n[GCC 7.5.0]" + "version": "3.9.17" }, "vscode": { "interpreter": { diff --git a/semilearn/lighting/trainer.py b/semilearn/lighting/trainer.py index afdf434de..780418aa9 100644 --- a/semilearn/lighting/trainer.py +++ b/semilearn/lighting/trainer.py @@ -28,16 +28,17 @@ def __init__(self, config, algorithm, verbose=0): def fit(self, train_lb_loader, train_ulb_loader, eval_loader): + self.algorithm.loader_dict = { + 'trian_lb': train_lb_loader, + 'train_ulb': train_ulb_loader, + 'eval': eval_loader + } self.algorithm.model.train() - - # EMA Init - self.algorithm.ema = EMA(self.algorithm.model, self.algorithm.ema_m) - self.algorithm.ema.register() - # train self.algorithm.it = 0 self.algorithm.best_eval_acc = 0.0 self.algorithm.best_epoch = 0 + self.algorithm.call_hook("before_run") for epoch in range(self.config.epoch): self.algorithm.epoch = epoch @@ -56,7 +57,9 @@ def fit(self, train_lb_loader, train_ulb_loader, eval_loader): break self.algorithm.call_hook("before_train_step") - result = self.algorithm.train_step(**self.algorithm.process_batch(**data_lb, **data_ulb)) + out_dict, log_dict = self.algorithm.train_step(**self.algorithm.process_batch(**data_lb, **data_ulb)) + self.algorithm.out_dict = out_dict + self.algorithm.log_dict = log_dict self.algorithm.call_hook("after_train_step") bar.suffix = ("Iter: {batch:4}/{iter:4}.".format(batch=self.algorithm.it, iter=len(train_lb_loader))) From 5b842fad8da36d7eadc04491b89cc0fd4ca75695 Mon Sep 17 00:00:00 2001 From: Hhhhhhao Date: Wed, 19 Jul 2023 23:48:18 -0400 Subject: [PATCH 10/12] [Update] release semilearn 0.3.1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index aa7a8d7f6..fe4904c12 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ setup( name='semilearn', - version='0.3.1a', + version='0.3.1', description='Unfied Semi-Supervised Learning Benchmark', long_description=long_description, long_description_content_type='text/markdown', From eb9ac290faf65945843a8927e13ef0083c883268 Mon Sep 17 00:00:00 2001 From: Hhhhhhao Date: Wed, 19 Jul 2023 23:50:42 -0400 Subject: [PATCH 11/12] [Update] auto issue closer --- .github/workflows/issue.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .github/workflows/issue.yml diff --git a/.github/workflows/issue.yml b/.github/workflows/issue.yml new file mode 100644 index 000000000..6d9376087 --- /dev/null +++ b/.github/workflows/issue.yml @@ -0,0 +1,17 @@ +name: 'Close stale issues and PR' +on: + schedule: + - cron: '30 1 * * *' + +jobs: + stale: + runs-on: ubuntu-latest + steps: + - uses: actions/stale@v8 + with: + stale-issue-message: 'This issue is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 5 days.' + stale-pr-message: 'This PR is stale because it has been open 45 days with no activity. Remove stale label or comment or this will be closed in 10 days.' + close-issue-message: 'This issue was closed because it has been stalled for 5 days with no activity.' + days-before-stale: 30 + days-before-close: 5 + days-before-pr-close: -1 \ No newline at end of file From 7fd120347212068b8c2b9e7f4256ecc2626a1f39 Mon Sep 17 00:00:00 2001 From: Hhhhhhao Date: Wed, 19 Jul 2023 23:51:46 -0400 Subject: [PATCH 12/12] [Update] auto issue closer --- .github/workflows/blank.yml | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/.github/workflows/blank.yml b/.github/workflows/blank.yml index 494ff3d63..61b10814e 100644 --- a/.github/workflows/blank.yml +++ b/.github/workflows/blank.yml @@ -33,22 +33,4 @@ jobs: - name: Run a multi-line script run: | echo Add other actions to build, - echo test, and deploy your project. - -name: 'Close stale issues and PR' -on: - schedule: - - cron: '30 1 * * *' - -jobs: - stale: - runs-on: ubuntu-latest - steps: - - uses: actions/stale@v8 - with: - stale-issue-message: 'This issue is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 5 days.' - stale-pr-message: 'This PR is stale because it has been open 45 days with no activity. Remove stale label or comment or this will be closed in 10 days.' - close-issue-message: 'This issue was closed because it has been stalled for 5 days with no activity.' - days-before-stale: 30 - days-before-close: 5 - days-before-pr-close: -1 \ No newline at end of file + echo test, and deploy your project. \ No newline at end of file