train.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4Q7nlrsOKfAj"
      },
      "source": [
        "## Training guide\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NZbgljOcM0if",
        "outputId": "0aaa2ddf-2709-4530-bee8-af1687710ad4"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "global:\n",
            "  username: nhtlong\n",
            "  project_name: \"zaloai-face-clf\"\n",
            "  name: \"faceB5-model\"\n",
            "  save_dir: ./runs2\n",
            "  use_fp16: true\n",
            "  debug: false\n",
            "  verbose: true\n",
            "  SEED: 1337\n",
            "  pretrained: null\n",
            "  resume: null\n",
            "  find_lr: false\n",
            "data:\n",
            "  name: ImageFolderFromCSV\n",
            "  args:\n",
            "    SIZE: 380\n",
            "    train:\n",
            "      CSV_PATH: data/train/labels_bbox_train.csv\n",
            "      IMG_DIR: data/train/faces/crop/\n",
            "      loader:\n",
            "        batch_size: 16\n",
            "        num_workers: 8\n",
            "        shuffle: True\n",
            "        drop_last: True\n",
            "    val:\n",
            "      CSV_PATH: data/train/labels_bbox_test.csv\n",
            "      IMG_DIR: data/train/faces/crop/\n",
            "      loader:\n",
            "        batch_size: 32\n",
            "        num_workers: 8\n",
            "        shuffle: False\n",
            "        drop_last: False\n",
            "extractors:\n",
            "  img_encoder:\n",
            "    name: EfficientNetExtractor\n",
            "    args:\n",
            "      version: 5\n",
            "      from_pretrained: True\n",
            "      freeze: False\n",
            "model:\n",
            "  name: FrameClassifier\n",
            "  args:\n",
            "    NUM_CLASS: 2 \n",
            "    EMBED_DIM: 1024\n",
            "metric:\n",
            "  - name: Accuracy\n",
            "    args:\n",
            "      label_key: \"labels\"\n",
            "trainer:\n",
            "  lr: 0.0004\n",
            "  num_epochs: 10000\n",
            "  clip_grad: 10.0\n",
            "  evaluate_interval: 1\n",
            "  print_interval: 20\n",
            "  save_interval: 1000\n",
            "  accumulate_grad_batches: 4\n",
            "callbacks:\n",
            "  - name: ModelCheckpoint\n",
            "    args:\n",
            "      filename: \"faceB5-{epoch}-{val/Accuracy:.4f}\"\n",
            "      monitor: \"val/Accuracy\"\n",
            "      verbose: True\n",
            "      save_top_k: 3\n",
            "      mode: max\n",
            "  - name: EarlyStopping\n",
            "    args:\n",
            "      monitor: \"val/Accuracy\"\n",
            "      min_delta: 0.0001\n",
            "      patience: 15\n",
            "      verbose: False\n",
            "      mode: max\n",
            "  - name: LearningRateMonitor\n",
            "    args:\n",
            "      logging_interval: step"
          ]
        }
      ],
      "source": [
        "!cat configs/faceb5.yml"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "B6C8153GMxKA",
        "outputId": "3eff3fbd-ad12-4fa6-daf4-1850fe8098f3"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n"
          ]
        }
      ],
      "source": [
        "WANDB_TOKEN=\"your token\"\n",
        "!wandb login --relogin $WANDB_TOKEN"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xPV1JRl7KiJJ",
        "outputId": "807aac74-b8f6-46d3-d18c-14c3dc2889d4"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Overriding configurating\n",
            "Global seed set to 1337\n",
            "Loaded pretrained weights for efficientnet-b5\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mnhtlong\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.13.5\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m./runs/wandb/run-20221113_181638-18bwolrr\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33mfaceB5-model-2022_11_13-18_16_36\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/nhtlong/zaloai-face-clf\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/nhtlong/zaloai-face-clf/runs/18bwolrr\u001b[0m\n",
            "Using 16bit native Automatic Mixed Precision (AMP)\n",
            "GPU available: True, used: True\n",
            "TPU available: False, using: 0 TPU cores\n",
            "IPU available: False, using: 0 IPUs\n",
            "HPU available: False, using: 0 HPUs\n",
            "You are using find_lr mode, the model will not be trained\n",
            "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
            "Accuracy: 0.73438 | \n",
            "\n",
            "Finding best initial lr: 100%|████████████████| 100/100 [02:44<00:00,  1.67s/it]Restoring states from the checkpoint path at runs/faceB5-model-2022_11_13-18_16_36/.lr_find_2ef7a08d-2065-4842-ae52-d888d5b489f7.ckpt\n",
            "/home/nhtlong/miniconda3/envs/zaloai/lib/python3.9/site-packages/pytorch_lightning/trainer/trainer.py:1721: UserWarning: Be aware that when using `ckpt_path`, callbacks used to create the checkpoint need to be provided during `Trainer` instantiation. Please add the following callbacks: [\"EarlyStopping{'monitor': 'val/Accuracy', 'mode': 'max'}\", \"ModelCheckpoint{'monitor': 'val/Accuracy', 'mode': 'max', 'every_n_train_steps': 0, 'every_n_epochs': 1, 'train_time_interval': None, 'save_on_train_epoch_end': None}\"].\n",
            "  rank_zero_warn(\n",
            "Finding best initial lr: 100%|████████████████| 100/100 [02:45<00:00,  1.66s/it]\n",
            "╒═══════════════╤══════════╕\n",
            "│ name          │    value │\n",
            "╞═══════════════╪══════════╡\n",
            "│ learning_rate │ 0.831764 │\n",
            "╘═══════════════╧══════════╛\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Waiting for W&B process to finish... \u001b[32m(success).\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Synced \u001b[33mfaceB5-model-2022_11_13-18_16_36\u001b[0m: \u001b[34m\u001b[4mhttps://wandb.ai/nhtlong/zaloai-face-clf/runs/18bwolrr\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./runs/wandb/run-20221113_181638-18bwolrr/logs\u001b[0m\n"
          ]
        }
      ],
      "source": [
        "# If you want to use find_lr mode, set CUDA_VISIBLE_DEVICES to 1 GPU\n",
        "!CUDA_VISIBLE_DEVICES=0 python scripts/train.py     -c configs/faceb5.yml \\\n",
        "                                                    -o  global.save_dir=./runs \\\n",
        "                                                        global.username=\"nhtlong\" \\\n",
        "                                                        global.find_lr=True "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Overriding configurating\n",
            "Global seed set to 1337\n",
            "Loaded pretrained weights for efficientnet-b5\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mnhtlong\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.13.5\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m./runs/wandb/run-20221113_182126-36twlug9\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33mfaceB5-model-2022_11_13-18_21_25\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/nhtlong/zaloai-face-clf\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/nhtlong/zaloai-face-clf/runs/36twlug9\u001b[0m\n",
            "Using 16bit native Automatic Mixed Precision (AMP)\n",
            "GPU available: True, used: True\n",
            "TPU available: False, using: 0 TPU cores\n",
            "IPU available: False, using: 0 IPUs\n",
            "HPU available: False, using: 0 HPUs\n",
            "If this is the first time you run this model, you can use global.find_lr=True to find the best lr\n",
            "Overriding configurating\n",
            "Global seed set to 1337\n",
            "Global seed set to 1337\n",
            "Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/2\n",
            "Loaded pretrained weights for efficientnet-b5\n",
            "If this is the first time you run this model, you can use global.find_lr=True to find the best lr\n",
            "Global seed set to 1337\n",
            "Initializing distributed: GLOBAL_RANK: 1, MEMBER: 2/2\n",
            "----------------------------------------------------------------------------------------------------\n",
            "distributed_backend=nccl\n",
            "All distributed processes registered. Starting with 2 processes\n",
            "----------------------------------------------------------------------------------------------------\n",
            "\n",
            "LOCAL_RANK: 1 - CUDA_VISIBLE_DEVICES: [0,1]\n",
            "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n",
            "\n",
            "  | Name         | Type                  | Params\n",
            "-------------------------------------------------------\n",
            "0 | visualExtrct | EfficientNetExtractor | 30.4 M\n",
            "1 | logits       | ClassifyBlock         | 2.1 M \n",
            "2 | loss         | FocalLoss             | 0     \n",
            "-------------------------------------------------------\n",
            "2.1 M     Trainable params\n",
            "30.4 M    Non-trainable params\n",
            "32.5 M    Total params\n",
            "64.984    Total estimated model params size (MB)\n",
            "Sanity Checking DataLoader 0:  95%|████████████▎| 19/20 [00:05<00:00,  3.27it/s]Accuracy: 0.49375 | \n",
            "\n",
            "Sanity Checking DataLoader 0: 100%|█████████████| 20/20 [00:06<00:00,  3.31it/s]Accuracy: 0.49219 | \n",
            "\n",
            "Epoch 0:   0%|        | 3/5342 [00:01<52:17,  1.70it/s, loss=0.0935, v_num=lug9][W reducer.cpp:1303] Warning: find_unused_parameters=True was specified in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the autograd graph every iteration,  which can adversely affect performance. If your model indeed never has any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator())\n",
            "[W reducer.cpp:1303] Warning: find_unused_parameters=True was specified in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the autograd graph every iteration,  which can adversely affect performance. If your model indeed never has any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator())\n",
            "Epoch 0:  97%|████▊| 5202/5342 [13:18<00:21,  6.51it/s, loss=0.0052, v_num=lug9]\n",
            "Validation: 0it [00:00, ?it/s]\u001b[A\n",
            "Validation DataLoader 0:   0%|                          | 0/140 [00:00<?, ?it/s]\u001b[A\n",
            "Epoch 0:  97%|████▊| 5203/5342 [13:20<00:21,  6.50it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  97%|████▊| 5204/5342 [13:20<00:21,  6.50it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  97%|████▊| 5205/5342 [13:20<00:21,  6.50it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  97%|████▊| 5206/5342 [13:21<00:20,  6.50it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  97%|████▊| 5207/5342 [13:21<00:20,  6.50it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  97%|████▊| 5208/5342 [13:21<00:20,  6.50it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5209/5342 [13:21<00:20,  6.50it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5210/5342 [13:22<00:20,  6.50it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5211/5342 [13:22<00:20,  6.50it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5212/5342 [13:22<00:20,  6.49it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5213/5342 [13:22<00:19,  6.49it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5214/5342 [13:22<00:19,  6.49it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5215/5342 [13:23<00:19,  6.49it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5216/5342 [13:23<00:19,  6.49it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5217/5342 [13:23<00:19,  6.49it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5218/5342 [13:23<00:19,  6.49it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5219/5342 [13:24<00:18,  6.49it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5220/5342 [13:24<00:18,  6.49it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5221/5342 [13:24<00:18,  6.49it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5222/5342 [13:24<00:18,  6.49it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5223/5342 [13:25<00:18,  6.49it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5224/5342 [13:25<00:18,  6.49it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5225/5342 [13:25<00:18,  6.49it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5226/5342 [13:25<00:17,  6.49it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5227/5342 [13:25<00:17,  6.49it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5228/5342 [13:26<00:17,  6.48it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5229/5342 [13:26<00:17,  6.48it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5230/5342 [13:26<00:17,  6.48it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5231/5342 [13:26<00:17,  6.48it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5232/5342 [13:27<00:16,  6.48it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5233/5342 [13:27<00:16,  6.48it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5234/5342 [13:27<00:16,  6.48it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5235/5342 [13:27<00:16,  6.48it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5236/5342 [13:28<00:16,  6.48it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5237/5342 [13:28<00:16,  6.48it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5238/5342 [13:28<00:16,  6.48it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5239/5342 [13:28<00:15,  6.48it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5240/5342 [13:28<00:15,  6.48it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5241/5342 [13:29<00:15,  6.48it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5242/5342 [13:29<00:15,  6.48it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5243/5342 [13:29<00:15,  6.48it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5244/5342 [13:29<00:15,  6.48it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5245/5342 [13:30<00:14,  6.47it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5246/5342 [13:30<00:14,  6.47it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5247/5342 [13:30<00:14,  6.47it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5248/5342 [13:30<00:14,  6.47it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5249/5342 [13:31<00:14,  6.47it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5250/5342 [13:31<00:14,  6.47it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5251/5342 [13:31<00:14,  6.47it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5252/5342 [13:31<00:13,  6.47it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5253/5342 [13:31<00:13,  6.47it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5254/5342 [13:32<00:13,  6.47it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5255/5342 [13:32<00:13,  6.47it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5256/5342 [13:32<00:13,  6.47it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5257/5342 [13:32<00:13,  6.47it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5258/5342 [13:33<00:12,  6.47it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5259/5342 [13:33<00:12,  6.47it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5260/5342 [13:33<00:12,  6.47it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  98%|████▉| 5261/5342 [13:33<00:12,  6.46it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5262/5342 [13:34<00:12,  6.46it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5263/5342 [13:34<00:12,  6.46it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5264/5342 [13:34<00:12,  6.46it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5265/5342 [13:34<00:11,  6.46it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5266/5342 [13:34<00:11,  6.46it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5267/5342 [13:35<00:11,  6.46it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5268/5342 [13:35<00:11,  6.46it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5269/5342 [13:35<00:11,  6.46it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5270/5342 [13:35<00:11,  6.46it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5271/5342 [13:36<00:10,  6.46it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5272/5342 [13:36<00:10,  6.46it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5273/5342 [13:36<00:10,  6.46it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5274/5342 [13:36<00:10,  6.46it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5275/5342 [13:37<00:10,  6.46it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5276/5342 [13:37<00:10,  6.46it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5277/5342 [13:37<00:10,  6.45it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5278/5342 [13:37<00:09,  6.45it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5279/5342 [13:38<00:09,  6.45it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5280/5342 [13:38<00:09,  6.45it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5281/5342 [13:38<00:09,  6.45it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5282/5342 [13:38<00:09,  6.45it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5283/5342 [13:38<00:09,  6.45it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5284/5342 [13:39<00:08,  6.45it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5285/5342 [13:39<00:08,  6.45it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5286/5342 [13:39<00:08,  6.45it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5287/5342 [13:39<00:08,  6.45it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5288/5342 [13:40<00:08,  6.45it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5289/5342 [13:40<00:08,  6.45it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5290/5342 [13:40<00:08,  6.45it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5291/5342 [13:40<00:07,  6.45it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5292/5342 [13:41<00:07,  6.45it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5293/5342 [13:41<00:07,  6.44it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5294/5342 [13:41<00:07,  6.44it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5295/5342 [13:41<00:07,  6.44it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5296/5342 [13:41<00:07,  6.44it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5297/5342 [13:42<00:06,  6.44it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5298/5342 [13:42<00:06,  6.44it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5299/5342 [13:42<00:06,  6.44it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5300/5342 [13:42<00:06,  6.44it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5301/5342 [13:43<00:06,  6.44it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5302/5342 [13:43<00:06,  6.44it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5303/5342 [13:43<00:06,  6.44it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5304/5342 [13:43<00:05,  6.44it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5305/5342 [13:44<00:05,  6.44it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5306/5342 [13:44<00:05,  6.44it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5307/5342 [13:44<00:05,  6.44it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5308/5342 [13:44<00:05,  6.44it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5309/5342 [13:44<00:05,  6.44it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5310/5342 [13:45<00:04,  6.43it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5311/5342 [13:45<00:04,  6.43it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5312/5342 [13:45<00:04,  6.43it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5313/5342 [13:45<00:04,  6.43it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5314/5342 [13:46<00:04,  6.43it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0:  99%|████▉| 5315/5342 [13:46<00:04,  6.43it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0: 100%|████▉| 5316/5342 [13:46<00:04,  6.43it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0: 100%|████▉| 5317/5342 [13:46<00:03,  6.43it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0: 100%|████▉| 5318/5342 [13:47<00:03,  6.43it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0: 100%|████▉| 5319/5342 [13:47<00:03,  6.43it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0: 100%|████▉| 5320/5342 [13:47<00:03,  6.43it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0: 100%|████▉| 5321/5342 [13:47<00:03,  6.43it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0: 100%|████▉| 5322/5342 [13:47<00:03,  6.43it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0: 100%|████▉| 5323/5342 [13:48<00:02,  6.43it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0: 100%|████▉| 5324/5342 [13:48<00:02,  6.43it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0: 100%|████▉| 5325/5342 [13:48<00:02,  6.43it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0: 100%|████▉| 5326/5342 [13:48<00:02,  6.43it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0: 100%|████▉| 5327/5342 [13:49<00:02,  6.42it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0: 100%|████▉| 5328/5342 [13:49<00:02,  6.42it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0: 100%|████▉| 5329/5342 [13:49<00:02,  6.42it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0: 100%|████▉| 5330/5342 [13:49<00:01,  6.42it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0: 100%|████▉| 5331/5342 [13:50<00:01,  6.42it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0: 100%|████▉| 5332/5342 [13:50<00:01,  6.42it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0: 100%|████▉| 5333/5342 [13:50<00:01,  6.42it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0: 100%|████▉| 5334/5342 [13:50<00:01,  6.42it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0: 100%|████▉| 5335/5342 [13:50<00:01,  6.42it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0: 100%|████▉| 5336/5342 [13:51<00:00,  6.42it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0: 100%|████▉| 5337/5342 [13:51<00:00,  6.42it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0: 100%|████▉| 5338/5342 [13:51<00:00,  6.42it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0: 100%|████▉| 5339/5342 [13:51<00:00,  6.42it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0: 100%|████▉| 5340/5342 [13:52<00:00,  6.42it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0: 100%|████▉| 5341/5342 [13:52<00:00,  6.42it/s, loss=0.0052, v_num=lug9]\u001b[A\n",
            "Epoch 0: 100%|█████| 5342/5342 [13:52<00:00,  6.42it/s, loss=0.0052, v_num=lug9]Accuracy: 0.89690 | \n",
            "\n",
            "Accuracy: 0.89960 | \n",
            "\n",
            "Epoch 0: 100%|█████| 5342/5342 [13:52<00:00,  6.42it/s, loss=0.0052, v_num=lug9]\n",
            "Epoch 0: 100%|█████| 5342/5342 [13:52<00:00,  6.42it/s, loss=0.0052, v_num=lug9]Epoch 0, global step 1301: 'val/Accuracy' reached 0.89825 (best 0.89825), saving model to './runs/zaloai-face-clf/36twlug9/checkpoints/faceB5-epoch=0-val/Accuracy=0.8982.ckpt' as top 3\n",
            "Epoch 1:   1%|   | 67/5342 [14:03<18:27:26, 12.60s/it, loss=0.00378, v_num=lug9]^C\n",
            "/home/nhtlong/miniconda3/envs/zaloai/lib/python3.9/site-packages/pytorch_lightning/trainer/trainer.py:727: UserWarning: Detected KeyboardInterrupt, attempting graceful shutdown...\n",
            "  rank_zero_warn(\"Detected KeyboardInterrupt, attempting graceful shutdown...\")\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Waiting for W&B process to finish... \u001b[32m(success).\u001b[0m\n"
          ]
        }
      ],
      "source": [
        "# The output of this cell is only for reference, because the backbone model is not trained. \n",
        "# Please ensure that enable extractors.img_encoder.args.freeze=False in configs/faceb5.yml\n",
        "!CUDA_VISIBLE_DEVICES=0,1 python scripts/train.py   -c configs/faceb5.yml \\\n",
        "                                                    -o  global.save_dir=./runs \\\n",
        "                                                        global.username=\"nhtlong\" \\\n",
        "                                                        global.find_lr=False \\\n",
        "                                                        trainer.lr=0.0001 \\\n",
        "                                                        trainer.num_epochs=2"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [],
      "name": "[AIC22][VER] Training Colors Classifier guide.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3.9.13 ('zaloai')",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.13"
    },
    "vscode": {
      "interpreter": {
        "hash": "20ae68c5166a594adc0b9bc0156f781891935baaf9f86f15a7c294214ff40d3d"
      }
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {}
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}