get_start2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "“Untitled0.ipynb”的副本",
      "provenance": [],
      "authorship_tag": "ABX9TyOTbUqAYKmYlfI5w1AmaGtk",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Jexxie/2021fall_dl/blob/main/%E2%80%9CUntitled0_ipynb%E2%80%9D%E7%9A%84%E5%89%AF%E6%9C%AC.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vF4j2YhIVW6m",
        "outputId": "91f39482-5af0-4e24-fe9f-f5af6f5b263b"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "import os"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "T73wdsNeXL34",
        "outputId": "86fd2b02-2ec5-4a8b-c562-637bd929e338"
      },
      "source": [
        "# %cd '/content/drive/My Drive' \n",
        "# !pwd "
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/content/drive/My Drive\n",
            "/content/drive/My Drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HGGJFbDXXPT6",
        "outputId": "f2662a11-aeb2-457f-c81a-7f280da97700"
      },
      "source": [
        "# !mkdir researchHub\n",
        "# %cd './researchHub'\n",
        "# # os.chdir('./researchHub')\n",
        "# !git clone https://github.com/princeton-nlp/SimCSE"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/content/drive/My Drive/researchHub\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZzEoTF3ysDbC",
        "outputId": "b9455152-b637-46f4-b07d-5272f961e974"
      },
      "source": [
        "import torch\n",
        "print(torch.__version__)\n",
        "print(torch.version.cuda)\n",
        "print(torch.cuda.get_device_name(0))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1.10.0+cu111\n",
            "11.1\n",
            "Tesla P100-PCIE-16GB\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uzi3KWltaMqQ",
        "outputId": "1c162968-ed90-464e-8329-5c213e068c6a"
      },
      "source": [
        "# %pwd\n",
        "# %cd SimCSE/\n",
        "# %ls"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/content/drive/My Drive/researchHub/SimCSE\n",
            "\u001b[0m\u001b[01;34mdata\u001b[0m/          README.md             setup.py\n",
            "\u001b[01;34mdemo\u001b[0m/          requirements.txt      \u001b[01;34msimcse\u001b[0m/\n",
            "evaluation.py  run_sup_example.sh    simcse_to_huggingface.py\n",
            "\u001b[01;34mfigure\u001b[0m/        run_unsup_example.sh  train.py\n",
            "LICENSE        \u001b[01;34mSentEval\u001b[0m/\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 486
        },
        "id": "cX7lsexbbo0x",
        "outputId": "c8870d57-2cd9-487b-e825-709926246abd"
      },
      "source": [
        "pip install simcse\n",
        "from simcse import SimCSE\n",
        "model = SimCSE(\"princeton-nlp/sup-simcse-bert-base-uncased\")\n",
        "embeddings = model.encode(\"A woman is reading.\")\n",
        "# print(embeddings)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "ename": "ModuleNotFoundError",
          "evalue": "ignored",
          "output_type": "error",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-8-afbdeddd5ba5>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0msimcse\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mSimCSE\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mSimCSE\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"princeton-nlp/sup-simcse-bert-base-uncased\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/content/drive/My Drive/researchHub/SimCSE/simcse/__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mtool\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mSimCSE\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/content/drive/My Drive/researchHub/SimCSE/simcse/tool.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtorch\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mtransformers\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtransformers\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mAutoModel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mAutoTokenizer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmetrics\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpairwise\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mcosine_similarity\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'transformers'",
            "",
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0;32m\nNOTE: If your import is failing due to a missing package, you can\nmanually install dependencies using either !pip or !apt.\n\nTo view examples of installing some common dependencies, click the\n\"Open Examples\" button below.\n\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uzCIBuZRb5J5",
        "outputId": "b68d5a6b-00af-45fb-c72e-4e01bc0806eb"
      },
      "source": [
        "sentences_a = ['A woman is reading.', 'A man is playing a guitar.']\n",
        "sentences_b = ['He plays guitar.', 'A woman is making a photo.']\n",
        "similarities = model.similarity(sentences_a, sentences_b)\n",
        "print(similarities)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 1/1 [00:00<00:00,  7.46it/s]\n",
            "100%|██████████| 1/1 [00:00<00:00,  7.10it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[[0.01262083 0.34469506]\n",
            " [0.89384234 0.04842842]]\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "n2-T1ohacxfN",
        "outputId": "2beda2da-6ef9-4829-e73d-df073afbd478"
      },
      "source": [
        "sentences = ['A woman is reading.', 'A man is playing a guitar.']\n",
        "model.build_index(sentences)\n",
        "results = model.search(\"He plays guitar.\")\n",
        "print(results)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "11/23/2021 21:16:25 - WARNING - simcse.tool -   Fail to import faiss. If you want to use faiss, install faiss through PyPI. Now the program continues with brute force search.\n",
            "11/23/2021 21:16:25 - INFO - simcse.tool -   Encoding embeddings for sentences...\n",
            "100%|██████████| 1/1 [00:00<00:00,  8.49it/s]\n",
            "11/23/2021 21:16:25 - INFO - simcse.tool -   Building index...\n",
            "11/23/2021 21:16:25 - INFO - simcse.tool -   Finished\n",
            "100%|██████████| 1/1 [00:00<00:00, 10.82it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[('A man is playing a guitar.', 0.893842339515686)]\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Xrggw_zuey7O",
        "outputId": "437faa97-d5b1-4c69-9a7d-8906a231c702"
      },
      "source": [
        "# pip install torch==1.7.1+cu110 -f https://download.pytorch.org/whl/torch_stable.html\n",
        "# %cd SentEval/data/downstream/\n",
        "# !bash download_dataset.sh\n",
        "# %pwd\n",
        "# %cd '/content/drive/My Drive/researchHub/SimCSE' \n",
        "# %ls\n",
        "!python evaluation.py \\\n",
        "    --model_name_or_path princeton-nlp/sup-simcse-bert-base-uncased \\\n",
        "    --pooler cls \\\n",
        "    --task_set sts \\\n",
        "    --mode test\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2021-11-23 21:17:27,241 : Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-11-23 21:17:27,388 : https://huggingface.co:443 \"HEAD /princeton-nlp/sup-simcse-bert-base-uncased/resolve/main/config.json HTTP/1.1\" 200 0\n",
            "2021-11-23 21:17:27,437 : Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-11-23 21:17:27,570 : https://huggingface.co:443 \"HEAD /princeton-nlp/sup-simcse-bert-base-uncased/resolve/main/pytorch_model.bin HTTP/1.1\" 302 0\n",
            "2021-11-23 21:17:29,164 : Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-11-23 21:17:29,300 : https://huggingface.co:443 \"HEAD /princeton-nlp/sup-simcse-bert-base-uncased/resolve/main/tokenizer_config.json HTTP/1.1\" 200 0\n",
            "2021-11-23 21:17:29,304 : Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-11-23 21:17:29,439 : https://huggingface.co:443 \"HEAD /princeton-nlp/sup-simcse-bert-base-uncased/resolve/main/config.json HTTP/1.1\" 200 0\n",
            "2021-11-23 21:17:29,445 : Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-11-23 21:17:29,575 : https://huggingface.co:443 \"GET /api/models/princeton-nlp/sup-simcse-bert-base-uncased HTTP/1.1\" 200 687\n",
            "2021-11-23 21:17:29,579 : Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-11-23 21:17:29,716 : https://huggingface.co:443 \"HEAD /princeton-nlp/sup-simcse-bert-base-uncased/resolve/main/vocab.txt HTTP/1.1\" 200 0\n",
            "2021-11-23 21:17:29,719 : Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-11-23 21:17:29,856 : https://huggingface.co:443 \"HEAD /princeton-nlp/sup-simcse-bert-base-uncased/resolve/main/tokenizer.json HTTP/1.1\" 404 0\n",
            "2021-11-23 21:17:29,859 : Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-11-23 21:17:29,989 : https://huggingface.co:443 \"HEAD /princeton-nlp/sup-simcse-bert-base-uncased/resolve/main/added_tokens.json HTTP/1.1\" 404 0\n",
            "2021-11-23 21:17:29,992 : Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-11-23 21:17:30,123 : https://huggingface.co:443 \"HEAD /princeton-nlp/sup-simcse-bert-base-uncased/resolve/main/special_tokens_map.json HTTP/1.1\" 200 0\n",
            "2021-11-23 21:17:30,126 : Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-11-23 21:17:30,256 : https://huggingface.co:443 \"HEAD /princeton-nlp/sup-simcse-bert-base-uncased/resolve/main/tokenizer_config.json HTTP/1.1\" 200 0\n",
            "2021-11-23 21:17:30,260 : Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-11-23 21:17:30,397 : https://huggingface.co:443 \"HEAD /princeton-nlp/sup-simcse-bert-base-uncased/resolve/main/config.json HTTP/1.1\" 200 0\n",
            "2021-11-23 21:17:30,608 : Starting new HTTPS connection (1): huggingface.co:443\n",
            "2021-11-23 21:17:30,736 : https://huggingface.co:443 \"HEAD /princeton-nlp/sup-simcse-bert-base-uncased/resolve/main/config.json HTTP/1.1\" 200 0\n",
            "2021-11-23 21:17:30,773 : ***** Transfer task : STS12 *****\n",
            "\n",
            "\n",
            "./SentEval/senteval/sts.py:42: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n",
            "  sent1 = np.array([s.split() for s in sent1])[not_empty_idx]\n",
            "./SentEval/senteval/sts.py:43: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n",
            "  sent2 = np.array([s.split() for s in sent2])[not_empty_idx]\n",
            "Traceback (most recent call last):\n",
            "  File \"evaluation.py\", line 205, in <module>\n",
            "    main()\n",
            "  File \"evaluation.py\", line 144, in main\n",
            "    result = se.eval(task)\n",
            "  File \"./SentEval/senteval/engine.py\", line 127, in eval\n",
            "    self.results = self.evaluation.run(self.params, self.batcher)\n",
            "  File \"./SentEval/senteval/sts.py\", line 72, in run\n",
            "    enc1 = batcher(params, batch1)\n",
            "  File \"evaluation.py\", line 114, in batcher\n",
            "    outputs = model(**batch, output_hidden_states=True, return_dict=True)\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py\", line 1102, in _call_impl\n",
            "    return forward_call(*input, **kwargs)\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/transformers/models/bert/modeling_bert.py\", line 1006, in forward\n",
            "    return_dict=return_dict,\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py\", line 1102, in _call_impl\n",
            "    return forward_call(*input, **kwargs)\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/transformers/models/bert/modeling_bert.py\", line 590, in forward\n",
            "    output_attentions,\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py\", line 1102, in _call_impl\n",
            "    return forward_call(*input, **kwargs)\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/transformers/models/bert/modeling_bert.py\", line 512, in forward\n",
            "    self.feed_forward_chunk, self.chunk_size_feed_forward, self.seq_len_dim, attention_output\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/transformers/modeling_utils.py\", line 2360, in apply_chunking_to_forward\n",
            "    return forward_fn(*input_tensors)\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/transformers/models/bert/modeling_bert.py\", line 524, in feed_forward_chunk\n",
            "    layer_output = self.output(intermediate_output, attention_output)\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py\", line 1102, in _call_impl\n",
            "    return forward_call(*input, **kwargs)\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/transformers/models/bert/modeling_bert.py\", line 437, in forward\n",
            "    hidden_states = self.dense(hidden_states)\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py\", line 1102, in _call_impl\n",
            "    return forward_call(*input, **kwargs)\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/torch/nn/modules/linear.py\", line 103, in forward\n",
            "    return F.linear(input, self.weight, self.bias)\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/torch/nn/functional.py\", line 1848, in linear\n",
            "    return torch._C._nn.linear(input, weight, bias)\n",
            "KeyboardInterrupt\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "I_XzGpNqdDXu",
        "outputId": "043f0dbd-8a64-4d91-d263-5206fa638ebd"
      },
      "source": [
        "%cd '/content/drive/My Drive/researchHub/SimCSE/data'\n",
        "!bash download_wiki.sh\n",
        "!bash download_nli.sh"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/content/drive/My Drive/researchHub/SimCSE/data\n",
            "--2021-11-23 21:22:20--  https://huggingface.co/datasets/princeton-nlp/datasets-for-simcse/resolve/main/wiki1m_for_simcse.txt\n",
            "Resolving huggingface.co (huggingface.co)... 54.221.222.133, 34.204.221.201, 35.173.116.130, ...\n",
            "Connecting to huggingface.co (huggingface.co)|54.221.222.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://cdn-lfs.huggingface.co/datasets/princeton-nlp/datasets-for-simcse/7b1825863a99aa76479b0456f7c210539dfaeeb69598b41fb4de4f524dd5a706 [following]\n",
            "--2021-11-23 21:22:21--  https://cdn-lfs.huggingface.co/datasets/princeton-nlp/datasets-for-simcse/7b1825863a99aa76479b0456f7c210539dfaeeb69598b41fb4de4f524dd5a706\n",
            "Resolving cdn-lfs.huggingface.co (cdn-lfs.huggingface.co)... 18.67.65.85, 18.67.65.19, 18.67.65.82, ...\n",
            "Connecting to cdn-lfs.huggingface.co (cdn-lfs.huggingface.co)|18.67.65.85|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 120038621 (114M) [text/plain]\n",
            "Saving to: ‘wiki1m_for_simcse.txt.1’\n",
            "\n",
            "wiki1m_for_simcse.t 100%[===================>] 114.48M  38.7MB/s    in 3.0s    \n",
            "\n",
            "2021-11-23 21:22:24 (38.7 MB/s) - ‘wiki1m_for_simcse.txt.1’ saved [120038621/120038621]\n",
            "\n",
            "--2021-11-23 21:22:24--  https://huggingface.co/datasets/princeton-nlp/datasets-for-simcse/resolve/main/nli_for_simcse.csv\n",
            "Resolving huggingface.co (huggingface.co)... 54.221.222.133, 34.204.221.201, 35.173.116.130, ...\n",
            "Connecting to huggingface.co (huggingface.co)|54.221.222.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://cdn-lfs.huggingface.co/datasets/princeton-nlp/datasets-for-simcse/0747687ec3594fa449d2004fd3757a56c24bf5f7428976fb5b67176775a68d48 [following]\n",
            "--2021-11-23 21:22:24--  https://cdn-lfs.huggingface.co/datasets/princeton-nlp/datasets-for-simcse/0747687ec3594fa449d2004fd3757a56c24bf5f7428976fb5b67176775a68d48\n",
            "Resolving cdn-lfs.huggingface.co (cdn-lfs.huggingface.co)... 52.85.149.60, 52.85.149.123, 52.85.149.11, ...\n",
            "Connecting to cdn-lfs.huggingface.co (cdn-lfs.huggingface.co)|52.85.149.60|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 48978197 (47M) [text/plain]\n",
            "Saving to: ‘nli_for_simcse.csv’\n",
            "\n",
            "nli_for_simcse.csv  100%[===================>]  46.71M  40.3MB/s    in 1.2s    \n",
            "\n",
            "2021-11-23 21:22:26 (40.3 MB/s) - ‘nli_for_simcse.csv’ saved [48978197/48978197]\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "SksF1WjT0oRi",
        "outputId": "5261fb34-a228-43c4-92a3-f708d71d0c02"
      },
      "source": [
        "!pip install -r requirements.txt"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Collecting transformers==4.2.1\n",
            "  Downloading transformers-4.2.1-py3-none-any.whl (1.8 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.8 MB 7.1 MB/s \n",
            "\u001b[?25hCollecting scipy==1.5.4\n",
            "  Downloading scipy-1.5.4-cp37-cp37m-manylinux1_x86_64.whl (25.9 MB)\n",
            "\u001b[K     |████████████████████████████████| 25.9 MB 1.2 MB/s \n",
            "\u001b[?25hCollecting datasets==1.2.1\n",
            "  Downloading datasets-1.2.1-py3-none-any.whl (159 kB)\n",
            "\u001b[K     |████████████████████████████████| 159 kB 74.8 MB/s \n",
            "\u001b[?25hRequirement already satisfied: pandas==1.1.5 in /usr/local/lib/python3.7/dist-packages (from -r requirements.txt (line 4)) (1.1.5)\n",
            "Collecting scikit-learn==0.24.0\n",
            "  Downloading scikit_learn-0.24.0-cp37-cp37m-manylinux2010_x86_64.whl (22.3 MB)\n",
            "\u001b[K     |████████████████████████████████| 22.3 MB 96.9 MB/s \n",
            "\u001b[?25hCollecting prettytable==2.1.0\n",
            "  Downloading prettytable-2.1.0-py3-none-any.whl (22 kB)\n",
            "Collecting gradio\n",
            "  Downloading gradio-2.4.6-py3-none-any.whl (979 kB)\n",
            "\u001b[K     |████████████████████████████████| 979 kB 61.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: torch in /usr/local/lib/python3.7/dist-packages (from -r requirements.txt (line 8)) (1.10.0+cu111)\n",
            "Collecting setuptools==49.3.0\n",
            "  Downloading setuptools-49.3.0-py3-none-any.whl (790 kB)\n",
            "\u001b[K     |████████████████████████████████| 790 kB 62.8 MB/s \n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers==4.2.1->-r requirements.txt (line 1)) (2019.12.20)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers==4.2.1->-r requirements.txt (line 1)) (21.3)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from transformers==4.2.1->-r requirements.txt (line 1)) (1.19.5)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers==4.2.1->-r requirements.txt (line 1)) (4.62.3)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers==4.2.1->-r requirements.txt (line 1)) (2.23.0)\n",
            "Collecting sacremoses\n",
            "  Downloading sacremoses-0.0.46-py3-none-any.whl (895 kB)\n",
            "\u001b[K     |████████████████████████████████| 895 kB 53.6 MB/s \n",
            "\u001b[?25hRequirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers==4.2.1->-r requirements.txt (line 1)) (4.8.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers==4.2.1->-r requirements.txt (line 1)) (3.4.0)\n",
            "Collecting tokenizers==0.9.4\n",
            "  Downloading tokenizers-0.9.4-cp37-cp37m-manylinux2010_x86_64.whl (2.9 MB)\n",
            "\u001b[K     |████████████████████████████████| 2.9 MB 57.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: pyarrow>=0.17.1 in /usr/local/lib/python3.7/dist-packages (from datasets==1.2.1->-r requirements.txt (line 3)) (3.0.0)\n",
            "Collecting tqdm>=4.27\n",
            "  Downloading tqdm-4.49.0-py2.py3-none-any.whl (69 kB)\n",
            "\u001b[K     |████████████████████████████████| 69 kB 9.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: multiprocess in /usr/local/lib/python3.7/dist-packages (from datasets==1.2.1->-r requirements.txt (line 3)) (0.70.12.2)\n",
            "Collecting xxhash\n",
            "  Downloading xxhash-2.0.2-cp37-cp37m-manylinux2010_x86_64.whl (243 kB)\n",
            "\u001b[K     |████████████████████████████████| 243 kB 70.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: dill in /usr/local/lib/python3.7/dist-packages (from datasets==1.2.1->-r requirements.txt (line 3)) (0.3.4)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas==1.1.5->-r requirements.txt (line 4)) (2018.9)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas==1.1.5->-r requirements.txt (line 4)) (2.8.2)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.24.0->-r requirements.txt (line 5)) (3.0.0)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.24.0->-r requirements.txt (line 5)) (1.1.0)\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.7/dist-packages (from prettytable==2.1.0->-r requirements.txt (line 6)) (0.2.5)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas==1.1.5->-r requirements.txt (line 4)) (1.15.0)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.2.1->-r requirements.txt (line 1)) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.2.1->-r requirements.txt (line 1)) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.2.1->-r requirements.txt (line 1)) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.2.1->-r requirements.txt (line 1)) (2021.10.8)\n",
            "Collecting flask-cachebuster\n",
            "  Downloading Flask-CacheBuster-1.0.0.tar.gz (3.1 kB)\n",
            "Collecting ffmpy\n",
            "  Downloading ffmpy-0.3.0.tar.gz (4.8 kB)\n",
            "Collecting pydub\n",
            "  Downloading pydub-0.25.1-py2.py3-none-any.whl (32 kB)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.7/dist-packages (from gradio->-r requirements.txt (line 7)) (3.2.2)\n",
            "Collecting pycryptodome\n",
            "  Downloading pycryptodome-3.11.0-cp35-abi3-manylinux2010_x86_64.whl (1.9 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.9 MB 57.3 MB/s \n",
            "\u001b[?25hCollecting markdown2\n",
            "  Downloading markdown2-2.4.1-py2.py3-none-any.whl (34 kB)\n",
            "Requirement already satisfied: Flask>=1.1.1 in /usr/local/lib/python3.7/dist-packages (from gradio->-r requirements.txt (line 7)) (1.1.4)\n",
            "Collecting Flask-Login\n",
            "  Downloading Flask_Login-0.5.0-py2.py3-none-any.whl (16 kB)\n",
            "Collecting analytics-python\n",
            "  Downloading analytics_python-1.4.0-py2.py3-none-any.whl (15 kB)\n",
            "Collecting paramiko\n",
            "  Downloading paramiko-2.8.0-py2.py3-none-any.whl (206 kB)\n",
            "\u001b[K     |████████████████████████████████| 206 kB 79.9 MB/s \n",
            "\u001b[?25hCollecting Flask-Cors>=3.0.8\n",
            "  Downloading Flask_Cors-3.0.10-py2.py3-none-any.whl (14 kB)\n",
            "Requirement already satisfied: pillow in /usr/local/lib/python3.7/dist-packages (from gradio->-r requirements.txt (line 7)) (7.1.2)\n",
            "Requirement already satisfied: Werkzeug<2.0,>=0.15 in /usr/local/lib/python3.7/dist-packages (from Flask>=1.1.1->gradio->-r requirements.txt (line 7)) (1.0.1)\n",
            "Requirement already satisfied: click<8.0,>=5.1 in /usr/local/lib/python3.7/dist-packages (from Flask>=1.1.1->gradio->-r requirements.txt (line 7)) (7.1.2)\n",
            "Requirement already satisfied: itsdangerous<2.0,>=0.24 in /usr/local/lib/python3.7/dist-packages (from Flask>=1.1.1->gradio->-r requirements.txt (line 7)) (1.1.0)\n",
            "Requirement already satisfied: Jinja2<3.0,>=2.10.1 in /usr/local/lib/python3.7/dist-packages (from Flask>=1.1.1->gradio->-r requirements.txt (line 7)) (2.11.3)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from Jinja2<3.0,>=2.10.1->Flask>=1.1.1->gradio->-r requirements.txt (line 7)) (2.0.1)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch->-r requirements.txt (line 8)) (3.10.0.2)\n",
            "Collecting backoff==1.10.0\n",
            "  Downloading backoff-1.10.0-py2.py3-none-any.whl (31 kB)\n",
            "Collecting monotonic>=1.5\n",
            "  Downloading monotonic-1.6-py2.py3-none-any.whl (8.2 kB)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers==4.2.1->-r requirements.txt (line 1)) (3.6.0)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->gradio->-r requirements.txt (line 7)) (3.0.6)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib->gradio->-r requirements.txt (line 7)) (0.11.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->gradio->-r requirements.txt (line 7)) (1.3.2)\n",
            "Collecting pynacl>=1.0.1\n",
            "  Downloading PyNaCl-1.4.0-cp35-abi3-manylinux1_x86_64.whl (961 kB)\n",
            "\u001b[K     |████████████████████████████████| 961 kB 67.2 MB/s \n",
            "\u001b[?25hCollecting cryptography>=2.5\n",
            "  Downloading cryptography-36.0.0-cp36-abi3-manylinux_2_24_x86_64.whl (3.6 MB)\n",
            "\u001b[K     |████████████████████████████████| 3.6 MB 72.9 MB/s \n",
            "\u001b[?25hCollecting bcrypt>=3.1.3\n",
            "  Downloading bcrypt-3.2.0-cp36-abi3-manylinux2010_x86_64.whl (63 kB)\n",
            "\u001b[K     |████████████████████████████████| 63 kB 3.4 MB/s \n",
            "\u001b[?25hRequirement already satisfied: cffi>=1.1 in /usr/local/lib/python3.7/dist-packages (from bcrypt>=3.1.3->paramiko->gradio->-r requirements.txt (line 7)) (1.15.0)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.7/dist-packages (from cffi>=1.1->bcrypt>=3.1.3->paramiko->gradio->-r requirements.txt (line 7)) (2.21)\n",
            "Building wheels for collected packages: ffmpy, flask-cachebuster\n",
            "  Building wheel for ffmpy (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for ffmpy: filename=ffmpy-0.3.0-py3-none-any.whl size=4710 sha256=b5af8284ea13f031a01ae05fcaaa07baa856e2430f1afa8912b0e4a5a0d6337e\n",
            "  Stored in directory: /root/.cache/pip/wheels/13/e4/6c/e8059816e86796a597c6e6b0d4c880630f51a1fcfa0befd5e6\n",
            "  Building wheel for flask-cachebuster (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for flask-cachebuster: filename=Flask_CacheBuster-1.0.0-py3-none-any.whl size=3371 sha256=d032f8bf2b612ea98b25a2d0c6e996bfc75229e22b72146b6042c318cedb85c1\n",
            "  Stored in directory: /root/.cache/pip/wheels/28/c0/c4/44687421dab41455be93112bd1b0dee1f3c5a9aa27bee63708\n",
            "Successfully built ffmpy flask-cachebuster\n",
            "Installing collected packages: tqdm, pynacl, monotonic, cryptography, bcrypt, backoff, xxhash, tokenizers, scipy, sacremoses, pydub, pycryptodome, paramiko, markdown2, Flask-Login, Flask-Cors, flask-cachebuster, ffmpy, analytics-python, transformers, setuptools, scikit-learn, prettytable, gradio, datasets\n",
            "  Attempting uninstall: tqdm\n",
            "    Found existing installation: tqdm 4.62.3\n",
            "    Uninstalling tqdm-4.62.3:\n",
            "      Successfully uninstalled tqdm-4.62.3\n",
            "  Attempting uninstall: scipy\n",
            "    Found existing installation: scipy 1.4.1\n",
            "    Uninstalling scipy-1.4.1:\n",
            "      Successfully uninstalled scipy-1.4.1\n",
            "  Attempting uninstall: setuptools\n",
            "    Found existing installation: setuptools 57.4.0\n",
            "    Uninstalling setuptools-57.4.0:\n",
            "      Successfully uninstalled setuptools-57.4.0\n",
            "  Attempting uninstall: scikit-learn\n",
            "    Found existing installation: scikit-learn 1.0.1\n",
            "    Uninstalling scikit-learn-1.0.1:\n",
            "      Successfully uninstalled scikit-learn-1.0.1\n",
            "  Attempting uninstall: prettytable\n",
            "    Found existing installation: prettytable 2.4.0\n",
            "    Uninstalling prettytable-2.4.0:\n",
            "      Successfully uninstalled prettytable-2.4.0\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "datascience 0.10.6 requires folium==0.2.1, but you have folium 0.8.3 which is incompatible.\n",
            "albumentations 0.1.12 requires imgaug<0.2.7,>=0.2.5, but you have imgaug 0.2.9 which is incompatible.\u001b[0m\n",
            "Successfully installed Flask-Cors-3.0.10 Flask-Login-0.5.0 analytics-python-1.4.0 backoff-1.10.0 bcrypt-3.2.0 cryptography-36.0.0 datasets-1.2.1 ffmpy-0.3.0 flask-cachebuster-1.0.0 gradio-2.4.6 markdown2-2.4.1 monotonic-1.6 paramiko-2.8.0 prettytable-2.1.0 pycryptodome-3.11.0 pydub-0.25.1 pynacl-1.4.0 sacremoses-0.0.46 scikit-learn-0.24.0 scipy-1.5.4 setuptools-49.3.0 tokenizers-0.9.4 tqdm-4.49.0 transformers-4.2.1 xxhash-2.0.2\n"
          ]
        },
        {
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "pkg_resources",
                  "tqdm"
                ]
              }
            }
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "background_save": true,
          "base_uri": "https://localhost:8080/"
        },
        "id": "yUg4kIvG1ZyM",
        "outputId": "10b96858-f45a-4d32-c116-4586803cbaf7"
      },
      "source": [
        "%cd '/content/drive/My Drive/researchHub/SimCSE' \n",
        "!sh run_unsup_example.sh"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/content/drive/My Drive/researchHub/SimCSE\n",
            "11/23/2021 21:33:00 - INFO - __main__ -   PyTorch: setting up devices\n",
            "11/23/2021 21:33:00 - WARNING - __main__ -   Process rank: -1, device: cuda:0, n_gpu: 1 distributed training: False, 16-bits training: True\n",
            "11/23/2021 21:33:00 - INFO - __main__ -   Training/evaluation parameters OurTrainingArguments(output_dir='result/my-unsup-simcse-bert-base-uncased', overwrite_output_dir=True, do_train=True, do_eval=True, do_predict=False, evaluation_strategy=<EvaluationStrategy.STEPS: 'steps'>, prediction_loss_only=False, per_device_train_batch_size=64, per_device_eval_batch_size=8, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=1, eval_accumulation_steps=None, learning_rate=3e-05, weight_decay=0.0, adam_beta1=0.9, adam_beta2=0.999, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=1.0, max_steps=-1, lr_scheduler_type=<SchedulerType.LINEAR: 'linear'>, warmup_steps=0, logging_dir='runs/Nov23_21-33-00_8fdefdf2218a', logging_first_step=False, logging_steps=500, save_steps=500, save_total_limit=None, no_cuda=False, seed=42, fp16=True, fp16_opt_level='O1', fp16_backend='auto', local_rank=-1, tpu_num_cores=None, tpu_metrics_debug=False, debug=False, dataloader_drop_last=False, eval_steps=125, dataloader_num_workers=0, past_index=-1, run_name='result/my-unsup-simcse-bert-base-uncased', disable_tqdm=False, remove_unused_columns=True, label_names=None, load_best_model_at_end=True, metric_for_best_model='stsb_spearman', greater_is_better=True, ignore_data_skip=False, sharded_ddp=False, deepspeed=None, label_smoothing_factor=0.0, adafactor=False, eval_transfer=False)\n",
            "Downloading: 2.57kB [00:00, 2.77MB/s]       \n",
            "Using custom data configuration default\n",
            "Downloading and preparing dataset text/default-de179036c6785ba0 (download: Unknown size, generated: Unknown size, post-processed: Unknown size, total: Unknown size) to ./data/text/default-de179036c6785ba0/0.0.0/daf90a707a433ac193b369c8cc1772139bb6cca21a9c7fe83bdd16aad9b9b6ab...\n",
            "Dataset text downloaded and prepared to ./data/text/default-de179036c6785ba0/0.0.0/daf90a707a433ac193b369c8cc1772139bb6cca21a9c7fe83bdd16aad9b9b6ab. Subsequent calls will reuse this data.\n",
            "[INFO|file_utils.py:1272] 2021-11-23 21:33:06,072 >> https://huggingface.co/bert-base-uncased/resolve/main/config.json not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmp8eau5h4a\n",
            "Downloading: 100% 570/570 [00:00<00:00, 606kB/s]\n",
            "[INFO|file_utils.py:1276] 2021-11-23 21:33:06,341 >> storing https://huggingface.co/bert-base-uncased/resolve/main/config.json in cache at /root/.cache/huggingface/transformers/3c61d016573b14f7f008c02c4e51a366c67ab274726fe2910691e2a761acf43e.37395cee442ab11005bcd270f3c34464dc1704b715b5d7d52b1a461abe3b9e4e\n",
            "[INFO|file_utils.py:1279] 2021-11-23 21:33:06,341 >> creating metadata file for /root/.cache/huggingface/transformers/3c61d016573b14f7f008c02c4e51a366c67ab274726fe2910691e2a761acf43e.37395cee442ab11005bcd270f3c34464dc1704b715b5d7d52b1a461abe3b9e4e\n",
            "[INFO|configuration_utils.py:445] 2021-11-23 21:33:06,342 >> loading configuration file https://huggingface.co/bert-base-uncased/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/3c61d016573b14f7f008c02c4e51a366c67ab274726fe2910691e2a761acf43e.37395cee442ab11005bcd270f3c34464dc1704b715b5d7d52b1a461abe3b9e4e\n",
            "[INFO|configuration_utils.py:481] 2021-11-23 21:33:06,342 >> Model config BertConfig {\n",
            "  \"architectures\": [\n",
            "    \"BertForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"gradient_checkpointing\": false,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"layer_norm_eps\": 1e-12,\n",
            "  \"max_position_embeddings\": 512,\n",
            "  \"model_type\": \"bert\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.2.1\",\n",
            "  \"type_vocab_size\": 2,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 30522\n",
            "}\n",
            "\n",
            "[INFO|configuration_utils.py:445] 2021-11-23 21:33:06,613 >> loading configuration file https://huggingface.co/bert-base-uncased/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/3c61d016573b14f7f008c02c4e51a366c67ab274726fe2910691e2a761acf43e.37395cee442ab11005bcd270f3c34464dc1704b715b5d7d52b1a461abe3b9e4e\n",
            "[INFO|configuration_utils.py:481] 2021-11-23 21:33:06,614 >> Model config BertConfig {\n",
            "  \"architectures\": [\n",
            "    \"BertForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"gradient_checkpointing\": false,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"layer_norm_eps\": 1e-12,\n",
            "  \"max_position_embeddings\": 512,\n",
            "  \"model_type\": \"bert\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.2.1\",\n",
            "  \"type_vocab_size\": 2,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 30522\n",
            "}\n",
            "\n",
            "[INFO|file_utils.py:1272] 2021-11-23 21:33:06,888 >> https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmpaf03xfrl\n",
            "Downloading: 100% 232k/232k [00:00<00:00, 924kB/s]\n",
            "[INFO|file_utils.py:1276] 2021-11-23 21:33:07,412 >> storing https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt in cache at /root/.cache/huggingface/transformers/45c3f7a79a80e1cf0a489e5c62b43f173c15db47864303a55d623bb3c96f72a5.d789d64ebfe299b0e416afc4a169632f903f693095b4629a7ea271d5a0cf2c99\n",
            "[INFO|file_utils.py:1279] 2021-11-23 21:33:07,412 >> creating metadata file for /root/.cache/huggingface/transformers/45c3f7a79a80e1cf0a489e5c62b43f173c15db47864303a55d623bb3c96f72a5.d789d64ebfe299b0e416afc4a169632f903f693095b4629a7ea271d5a0cf2c99\n",
            "[INFO|file_utils.py:1272] 2021-11-23 21:33:07,683 >> https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmpd10nlbi_\n",
            "Downloading: 100% 466k/466k [00:00<00:00, 1.47MB/s]\n",
            "[INFO|file_utils.py:1276] 2021-11-23 21:33:08,278 >> storing https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json in cache at /root/.cache/huggingface/transformers/534479488c54aeaf9c3406f647aa2ec13648c06771ffe269edabebd4c412da1d.7f2721073f19841be16f41b0a70b600ca6b880c8f3df6f3535cbc704371bdfa4\n",
            "[INFO|file_utils.py:1279] 2021-11-23 21:33:08,278 >> creating metadata file for /root/.cache/huggingface/transformers/534479488c54aeaf9c3406f647aa2ec13648c06771ffe269edabebd4c412da1d.7f2721073f19841be16f41b0a70b600ca6b880c8f3df6f3535cbc704371bdfa4\n",
            "[INFO|tokenization_utils_base.py:1766] 2021-11-23 21:33:08,278 >> loading file https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt from cache at /root/.cache/huggingface/transformers/45c3f7a79a80e1cf0a489e5c62b43f173c15db47864303a55d623bb3c96f72a5.d789d64ebfe299b0e416afc4a169632f903f693095b4629a7ea271d5a0cf2c99\n",
            "[INFO|tokenization_utils_base.py:1766] 2021-11-23 21:33:08,278 >> loading file https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json from cache at /root/.cache/huggingface/transformers/534479488c54aeaf9c3406f647aa2ec13648c06771ffe269edabebd4c412da1d.7f2721073f19841be16f41b0a70b600ca6b880c8f3df6f3535cbc704371bdfa4\n",
            "[INFO|file_utils.py:1272] 2021-11-23 21:33:08,568 >> https://huggingface.co/bert-base-uncased/resolve/main/pytorch_model.bin not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmpzxx6glow\n",
            "Downloading: 100% 440M/440M [00:07<00:00, 57.4MB/s]\n",
            "[INFO|file_utils.py:1276] 2021-11-23 21:33:16,332 >> storing https://huggingface.co/bert-base-uncased/resolve/main/pytorch_model.bin in cache at /root/.cache/huggingface/transformers/a8041bf617d7f94ea26d15e218abd04afc2004805632abc0ed2066aa16d50d04.faf6ea826ae9c5867d12b22257f9877e6b8367890837bd60f7c54a29633f7f2f\n",
            "[INFO|file_utils.py:1279] 2021-11-23 21:33:16,333 >> creating metadata file for /root/.cache/huggingface/transformers/a8041bf617d7f94ea26d15e218abd04afc2004805632abc0ed2066aa16d50d04.faf6ea826ae9c5867d12b22257f9877e6b8367890837bd60f7c54a29633f7f2f\n",
            "[INFO|modeling_utils.py:1027] 2021-11-23 21:33:16,333 >> loading weights file https://huggingface.co/bert-base-uncased/resolve/main/pytorch_model.bin from cache at /root/.cache/huggingface/transformers/a8041bf617d7f94ea26d15e218abd04afc2004805632abc0ed2066aa16d50d04.faf6ea826ae9c5867d12b22257f9877e6b8367890837bd60f7c54a29633f7f2f\n",
            "[WARNING|modeling_utils.py:1135] 2021-11-23 21:33:19,172 >> Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForCL: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'bert.pooler.dense.weight', 'bert.pooler.dense.bias']\n",
            "- This IS expected if you are initializing BertForCL from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForCL from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "[WARNING|modeling_utils.py:1146] 2021-11-23 21:33:19,172 >> Some weights of BertForCL were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['mlp.dense.weight', 'mlp.dense.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "100% 1001/1001 [02:42<00:00,  6.15ba/s]\n",
            "/usr/local/lib/python3.7/dist-packages/_distutils_hack/__init__.py:19: UserWarning: Distutils was imported before Setuptools. This usage is discouraged and may exhibit undesirable behaviors or errors. Please use Setuptools' objects directly or at least import Setuptools first.\n",
            "  \"Distutils was imported before Setuptools. This usage is discouraged \"\n",
            "[INFO|trainer.py:442] 2021-11-23 21:36:13,349 >> The following columns in the training set don't have a corresponding argument in `BertForCL.forward` and have been ignored: .\n",
            "[INFO|trainer.py:358] 2021-11-23 21:36:13,349 >> Using amp fp16 backend\n",
            "11/23/2021 21:36:13 - INFO - simcse.trainers -   ***** Running training *****\n",
            "11/23/2021 21:36:13 - INFO - simcse.trainers -     Num examples = 1000001\n",
            "11/23/2021 21:36:13 - INFO - simcse.trainers -     Num Epochs = 1\n",
            "11/23/2021 21:36:13 - INFO - simcse.trainers -     Instantaneous batch size per device = 64\n",
            "11/23/2021 21:36:13 - INFO - simcse.trainers -     Total train batch size (w. parallel, distributed & accumulation) = 64\n",
            "11/23/2021 21:36:13 - INFO - simcse.trainers -     Gradient Accumulation steps = 1\n",
            "11/23/2021 21:36:13 - INFO - simcse.trainers -     Total optimization steps = 15626\n",
            "  0% 0/15626 [00:00<?, ?it/s]/usr/local/lib/python3.7/dist-packages/torch/optim/lr_scheduler.py:134: UserWarning: Detected call of `lr_scheduler.step()` before `optimizer.step()`. In PyTorch 1.1.0 and later, you should call them in the opposite order: `optimizer.step()` before `lr_scheduler.step()`.  Failure to do this will result in PyTorch skipping the first value of the learning rate schedule. See more details at https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\n",
            "  \"https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\", UserWarning)\n",
            "{'eval_stsb_spearman': 0.6310960417267056, 'eval_sickr_spearman': 0.6258643200858789, 'eval_avg_sts': 0.6284801809062923, 'epoch': 0.01}\n",
            "  1% 125/15626 [01:30<1:43:14,  2.50it/s][INFO|trainer.py:1344] 2021-11-23 21:37:44,334 >> Saving model checkpoint to result/my-unsup-simcse-bert-base-uncased\n",
            "[INFO|configuration_utils.py:300] 2021-11-23 21:37:44,341 >> Configuration saved in result/my-unsup-simcse-bert-base-uncased/config.json\n",
            "[INFO|modeling_utils.py:817] 2021-11-23 21:37:46,229 >> Model weights saved in result/my-unsup-simcse-bert-base-uncased/pytorch_model.bin\n",
            "{'eval_stsb_spearman': 0.6636022901093679, 'eval_sickr_spearman': 0.6361891336821169, 'eval_avg_sts': 0.6498957118957425, 'epoch': 0.02}\n",
            "  2% 250/15626 [03:01<1:43:05,  2.49it/s][INFO|trainer.py:1344] 2021-11-23 21:39:15,243 >> Saving model checkpoint to result/my-unsup-simcse-bert-base-uncased\n",
            "[INFO|configuration_utils.py:300] 2021-11-23 21:39:15,249 >> Configuration saved in result/my-unsup-simcse-bert-base-uncased/config.json\n",
            "[INFO|modeling_utils.py:817] 2021-11-23 21:39:17,080 >> Model weights saved in result/my-unsup-simcse-bert-base-uncased/pytorch_model.bin\n",
            "{'eval_stsb_spearman': 0.7331321102728275, 'eval_sickr_spearman': 0.6798508458224045, 'eval_avg_sts': 0.706491478047616, 'epoch': 0.02}\n",
            "  2% 375/15626 [04:33<1:42:24,  2.48it/s][INFO|trainer.py:1344] 2021-11-23 21:40:47,161 >> Saving model checkpoint to result/my-unsup-simcse-bert-base-uncased\n",
            "[INFO|configuration_utils.py:300] 2021-11-23 21:40:47,169 >> Configuration saved in result/my-unsup-simcse-bert-base-uncased/config.json\n",
            "[INFO|modeling_utils.py:817] 2021-11-23 21:40:49,284 >> Model weights saved in result/my-unsup-simcse-bert-base-uncased/pytorch_model.bin\n",
            "{'loss': 0.0098, 'learning_rate': 2.9040061436068092e-05, 'epoch': 0.03}\n",
            "{'eval_stsb_spearman': 0.7615211945730501, 'eval_sickr_spearman': 0.6981415215299508, 'eval_avg_sts': 0.7298313580515005, 'epoch': 0.03}\n",
            "  3% 500/15626 [06:04<1:39:42,  2.53it/s][INFO|trainer.py:1344] 2021-11-23 21:42:17,630 >> Saving model checkpoint to result/my-unsup-simcse-bert-base-uncased\n",
            "[INFO|configuration_utils.py:300] 2021-11-23 21:42:17,640 >> Configuration saved in result/my-unsup-simcse-bert-base-uncased/config.json\n",
            "[INFO|modeling_utils.py:817] 2021-11-23 21:42:19,801 >> Model weights saved in result/my-unsup-simcse-bert-base-uncased/pytorch_model.bin\n",
            "{'eval_stsb_spearman': 0.7563857604240498, 'eval_sickr_spearman': 0.7029344490802572, 'eval_avg_sts': 0.7296601047521535, 'epoch': 0.04}\n",
            "{'eval_stsb_spearman': 0.7547053600170259, 'eval_sickr_spearman': 0.6982280255436408, 'eval_avg_sts': 0.7264666927803334, 'epoch': 0.05}\n",
            "{'eval_stsb_spearman': 0.7521070966604101, 'eval_sickr_spearman': 0.7060045010219784, 'eval_avg_sts': 0.7290557988411943, 'epoch': 0.06}\n",
            "{'loss': 0.0003, 'learning_rate': 2.8080122872136184e-05, 'epoch': 0.06}\n",
            "{'eval_stsb_spearman': 0.7768643884355731, 'eval_sickr_spearman': 0.7200967781528193, 'eval_avg_sts': 0.7484805832941962, 'epoch': 0.06}\n",
            "  6% 1000/15626 [11:42<1:35:58,  2.54it/s][INFO|trainer.py:1344] 2021-11-23 21:47:55,536 >> Saving model checkpoint to result/my-unsup-simcse-bert-base-uncased\n",
            "[INFO|configuration_utils.py:300] 2021-11-23 21:47:55,542 >> Configuration saved in result/my-unsup-simcse-bert-base-uncased/config.json\n",
            "[INFO|modeling_utils.py:817] 2021-11-23 21:47:57,437 >> Model weights saved in result/my-unsup-simcse-bert-base-uncased/pytorch_model.bin\n",
            "{'eval_stsb_spearman': 0.7795047472160654, 'eval_sickr_spearman': 0.7212496206495369, 'eval_avg_sts': 0.7503771839328012, 'epoch': 0.07}\n",
            "  7% 1125/15626 [13:11<1:35:08,  2.54it/s][INFO|trainer.py:1344] 2021-11-23 21:49:25,182 >> Saving model checkpoint to result/my-unsup-simcse-bert-base-uncased\n",
            "[INFO|configuration_utils.py:300] 2021-11-23 21:49:25,188 >> Configuration saved in result/my-unsup-simcse-bert-base-uncased/config.json\n",
            "[INFO|modeling_utils.py:817] 2021-11-23 21:49:27,047 >> Model weights saved in result/my-unsup-simcse-bert-base-uncased/pytorch_model.bin\n",
            "{'eval_stsb_spearman': 0.7950770042071407, 'eval_sickr_spearman': 0.7156211920897144, 'eval_avg_sts': 0.7553490981484275, 'epoch': 0.08}\n",
            "  8% 1250/15626 [14:41<1:35:19,  2.51it/s][INFO|trainer.py:1344] 2021-11-23 21:50:55,017 >> Saving model checkpoint to result/my-unsup-simcse-bert-base-uncased\n",
            "[INFO|configuration_utils.py:300] 2021-11-23 21:50:55,023 >> Configuration saved in result/my-unsup-simcse-bert-base-uncased/config.json\n",
            "[INFO|modeling_utils.py:817] 2021-11-23 21:50:56,811 >> Model weights saved in result/my-unsup-simcse-bert-base-uncased/pytorch_model.bin\n",
            "{'eval_stsb_spearman': 0.7996905298118206, 'eval_sickr_spearman': 0.7159029425307505, 'eval_avg_sts': 0.7577967361712856, 'epoch': 0.09}\n",
            "  9% 1375/15626 [16:10<1:34:28,  2.51it/s][INFO|trainer.py:1344] 2021-11-23 21:52:24,145 >> Saving model checkpoint to result/my-unsup-simcse-bert-base-uncased\n",
            "[INFO|configuration_utils.py:300] 2021-11-23 21:52:24,151 >> Configuration saved in result/my-unsup-simcse-bert-base-uncased/config.json\n",
            "[INFO|modeling_utils.py:817] 2021-11-23 21:52:25,918 >> Model weights saved in result/my-unsup-simcse-bert-base-uncased/pytorch_model.bin\n",
            "{'loss': 0.0001, 'learning_rate': 2.7120184308204276e-05, 'epoch': 0.1}\n",
            "{'eval_stsb_spearman': 0.8036633795250842, 'eval_sickr_spearman': 0.7185667474165082, 'eval_avg_sts': 0.7611150634707962, 'epoch': 0.1}\n",
            " 10% 1500/15626 [17:39<1:33:31,  2.52it/s][INFO|trainer.py:1344] 2021-11-23 21:53:53,249 >> Saving model checkpoint to result/my-unsup-simcse-bert-base-uncased\n",
            "[INFO|configuration_utils.py:300] 2021-11-23 21:53:53,255 >> Configuration saved in result/my-unsup-simcse-bert-base-uncased/config.json\n",
            "[INFO|modeling_utils.py:817] 2021-11-23 21:53:55,300 >> Model weights saved in result/my-unsup-simcse-bert-base-uncased/pytorch_model.bin\n",
            "{'eval_stsb_spearman': 0.8042022072985913, 'eval_sickr_spearman': 0.7197752099286912, 'eval_avg_sts': 0.7619887086136412, 'epoch': 0.1}\n",
            " 10% 1625/15626 [19:09<1:31:53,  2.54it/s][INFO|trainer.py:1344] 2021-11-23 21:55:22,530 >> Saving model checkpoint to result/my-unsup-simcse-bert-base-uncased\n",
            "[INFO|configuration_utils.py:300] 2021-11-23 21:55:22,537 >> Configuration saved in result/my-unsup-simcse-bert-base-uncased/config.json\n",
            "[INFO|modeling_utils.py:817] 2021-11-23 21:55:24,294 >> Model weights saved in result/my-unsup-simcse-bert-base-uncased/pytorch_model.bin\n",
            "{'eval_stsb_spearman': 0.8078293083813073, 'eval_sickr_spearman': 0.7165279712537646, 'eval_avg_sts': 0.7621786398175359, 'epoch': 0.11}\n",
            " 11% 1750/15626 [20:38<1:31:09,  2.54it/s][INFO|trainer.py:1344] 2021-11-23 21:56:51,726 >> Saving model checkpoint to result/my-unsup-simcse-bert-base-uncased\n",
            "[INFO|configuration_utils.py:300] 2021-11-23 21:56:51,733 >> Configuration saved in result/my-unsup-simcse-bert-base-uncased/config.json\n",
            "[INFO|modeling_utils.py:817] 2021-11-23 21:56:53,616 >> Model weights saved in result/my-unsup-simcse-bert-base-uncased/pytorch_model.bin\n",
            "{'eval_stsb_spearman': 0.8061911008450688, 'eval_sickr_spearman': 0.7196979759175786, 'eval_avg_sts': 0.7629445383813237, 'epoch': 0.12}\n",
            "{'loss': 0.0001, 'learning_rate': 2.6160245744272367e-05, 'epoch': 0.13}\n",
            "{'eval_stsb_spearman': 0.7990132179571022, 'eval_sickr_spearman': 0.7173561235047605, 'eval_avg_sts': 0.7581846707309314, 'epoch': 0.13}\n",
            "{'eval_stsb_spearman': 0.8078664062716778, 'eval_sickr_spearman': 0.7200063836199126, 'eval_avg_sts': 0.7639363949457952, 'epoch': 0.14}\n",
            " 14% 2125/15626 [24:52<1:28:27,  2.54it/s][INFO|trainer.py:1344] 2021-11-23 22:01:05,932 >> Saving model checkpoint to result/my-unsup-simcse-bert-base-uncased\n",
            "[INFO|configuration_utils.py:300] 2021-11-23 22:01:05,938 >> Configuration saved in result/my-unsup-simcse-bert-base-uncased/config.json\n",
            "[INFO|modeling_utils.py:817] 2021-11-23 22:01:07,745 >> Model weights saved in result/my-unsup-simcse-bert-base-uncased/pytorch_model.bin\n",
            "{'eval_stsb_spearman': 0.7836420852837526, 'eval_sickr_spearman': 0.7070913968164214, 'eval_avg_sts': 0.745366741050087, 'epoch': 0.14}\n",
            "{'eval_stsb_spearman': 0.813954698729501, 'eval_sickr_spearman': 0.7230127943522285, 'eval_avg_sts': 0.7684837465408647, 'epoch': 0.15}\n",
            " 15% 2375/15626 [27:44<1:27:04,  2.54it/s][INFO|trainer.py:1344] 2021-11-23 22:03:57,756 >> Saving model checkpoint to result/my-unsup-simcse-bert-base-uncased\n",
            "[INFO|configuration_utils.py:300] 2021-11-23 22:03:57,762 >> Configuration saved in result/my-unsup-simcse-bert-base-uncased/config.json\n",
            "[INFO|modeling_utils.py:817] 2021-11-23 22:03:59,591 >> Model weights saved in result/my-unsup-simcse-bert-base-uncased/pytorch_model.bin\n",
            "{'loss': 0.0002, 'learning_rate': 2.5200307180340456e-05, 'epoch': 0.16}\n",
            "{'eval_stsb_spearman': 0.8122608354609038, 'eval_sickr_spearman': 0.7228795560768392, 'eval_avg_sts': 0.7675701957688714, 'epoch': 0.16}\n",
            "{'eval_stsb_spearman': 0.8137731551762357, 'eval_sickr_spearman': 0.7234037675179347, 'eval_avg_sts': 0.7685884613470852, 'epoch': 0.17}\n",
            "{'eval_stsb_spearman': 0.8179668626929649, 'eval_sickr_spearman': 0.7256945628699242, 'eval_avg_sts': 0.7718307127814446, 'epoch': 0.18}\n",
            " 18% 2750/15626 [31:59<1:24:16,  2.55it/s][INFO|trainer.py:1344] 2021-11-23 22:08:12,512 >> Saving model checkpoint to result/my-unsup-simcse-bert-base-uncased\n",
            "[INFO|configuration_utils.py:300] 2021-11-23 22:08:12,519 >> Configuration saved in result/my-unsup-simcse-bert-base-uncased/config.json\n",
            "[INFO|modeling_utils.py:817] 2021-11-23 22:08:14,344 >> Model weights saved in result/my-unsup-simcse-bert-base-uncased/pytorch_model.bin\n",
            "{'eval_stsb_spearman': 0.8213321397638124, 'eval_sickr_spearman': 0.737021161148857, 'eval_avg_sts': 0.7791766504563347, 'epoch': 0.18}\n",
            " 18% 2875/15626 [33:28<1:23:49,  2.54it/s][INFO|trainer.py:1344] 2021-11-23 22:09:41,549 >> Saving model checkpoint to result/my-unsup-simcse-bert-base-uncased\n",
            "[INFO|configuration_utils.py:300] 2021-11-23 22:09:41,556 >> Configuration saved in result/my-unsup-simcse-bert-base-uncased/config.json\n",
            "[INFO|modeling_utils.py:817] 2021-11-23 22:09:43,439 >> Model weights saved in result/my-unsup-simcse-bert-base-uncased/pytorch_model.bin\n",
            "{'loss': 0.0002, 'learning_rate': 2.424036861640855e-05, 'epoch': 0.19}\n",
            "{'eval_stsb_spearman': 0.8344697352770686, 'eval_sickr_spearman': 0.7453583997364845, 'eval_avg_sts': 0.7899140675067766, 'epoch': 0.19}\n",
            " 19% 3000/15626 [34:57<1:22:50,  2.54it/s][INFO|trainer.py:1344] 2021-11-23 22:11:10,662 >> Saving model checkpoint to result/my-unsup-simcse-bert-base-uncased\n",
            "[INFO|configuration_utils.py:300] 2021-11-23 22:11:10,668 >> Configuration saved in result/my-unsup-simcse-bert-base-uncased/config.json\n",
            "[INFO|modeling_utils.py:817] 2021-11-23 22:11:12,507 >> Model weights saved in result/my-unsup-simcse-bert-base-uncased/pytorch_model.bin\n",
            "{'eval_stsb_spearman': 0.8210370498623856, 'eval_sickr_spearman': 0.734373782899791, 'eval_avg_sts': 0.7777054163810884, 'epoch': 0.2}\n",
            "{'eval_stsb_spearman': 0.8223342507756645, 'eval_sickr_spearman': 0.7398749290408024, 'eval_avg_sts': 0.7811045899082334, 'epoch': 0.21}\n",
            "{'eval_stsb_spearman': 0.8236671904651712, 'eval_sickr_spearman': 0.7416484294303033, 'eval_avg_sts': 0.7826578099477373, 'epoch': 0.22}\n",
            "{'loss': 0.0001, 'learning_rate': 2.3280430052476642e-05, 'epoch': 0.22}\n",
            "{'eval_stsb_spearman': 0.825915281716723, 'eval_sickr_spearman': 0.7451920680322044, 'eval_avg_sts': 0.7855536748744637, 'epoch': 0.22}\n",
            "{'eval_stsb_spearman': 0.8236893346085474, 'eval_sickr_spearman': 0.7447561857866535, 'eval_avg_sts': 0.7842227601976004, 'epoch': 0.23}\n",
            "{'eval_stsb_spearman': 0.8193925583541122, 'eval_sickr_spearman': 0.7346561577451457, 'eval_avg_sts': 0.777024358049629, 'epoch': 0.24}\n",
            "{'eval_stsb_spearman': 0.7926416466545882, 'eval_sickr_spearman': 0.7190067603367828, 'eval_avg_sts': 0.7558242034956855, 'epoch': 0.25}\n",
            "{'loss': 0.0001, 'learning_rate': 2.2320491488544734e-05, 'epoch': 0.26}\n",
            "{'eval_stsb_spearman': 0.801041000493484, 'eval_sickr_spearman': 0.7252434547652173, 'eval_avg_sts': 0.7631422276293507, 'epoch': 0.26}\n",
            "{'eval_stsb_spearman': 0.8049552245473377, 'eval_sickr_spearman': 0.7343892489144541, 'eval_avg_sts': 0.7696722367308959, 'epoch': 0.26}\n",
            "{'eval_stsb_spearman': 0.8084491052434394, 'eval_sickr_spearman': 0.7376621842286499, 'eval_avg_sts': 0.7730556447360446, 'epoch': 0.27}\n",
            "{'eval_stsb_spearman': 0.8128532524069875, 'eval_sickr_spearman': 0.7394112848186203, 'eval_avg_sts': 0.7761322686128038, 'epoch': 0.28}\n",
            "{'loss': 0.0001, 'learning_rate': 2.1360552924612826e-05, 'epoch': 0.29}\n",
            "{'eval_stsb_spearman': 0.8253300955258724, 'eval_sickr_spearman': 0.7367387382724008, 'eval_avg_sts': 0.7810344168991366, 'epoch': 0.29}\n",
            "{'eval_stsb_spearman': 0.7974718030128417, 'eval_sickr_spearman': 0.7203442343874285, 'eval_avg_sts': 0.7589080187001351, 'epoch': 0.3}\n",
            "{'eval_stsb_spearman': 0.8004982741407775, 'eval_sickr_spearman': 0.7204803545289039, 'eval_avg_sts': 0.7604893143348407, 'epoch': 0.3}\n",
            "{'eval_stsb_spearman': 0.7855704062009201, 'eval_sickr_spearman': 0.7087643681106124, 'eval_avg_sts': 0.7471673871557662, 'epoch': 0.31}\n",
            "{'loss': 0.0002, 'learning_rate': 2.0400614360680917e-05, 'epoch': 0.32}\n",
            "{'eval_stsb_spearman': 0.7924545805039171, 'eval_sickr_spearman': 0.7134096480550981, 'eval_avg_sts': 0.7529321142795076, 'epoch': 0.32}\n",
            "{'eval_stsb_spearman': 0.8123364175312007, 'eval_sickr_spearman': 0.7287498692635039, 'eval_avg_sts': 0.7705431433973523, 'epoch': 0.33}\n",
            "{'eval_stsb_spearman': 0.8101106971359757, 'eval_sickr_spearman': 0.7301961337589068, 'eval_avg_sts': 0.7701534154474412, 'epoch': 0.34}\n",
            "{'eval_stsb_spearman': 0.8100892878685508, 'eval_sickr_spearman': 0.7296069537482027, 'eval_avg_sts': 0.7698481208083767, 'epoch': 0.34}\n",
            "{'loss': 0.0002, 'learning_rate': 1.944067579674901e-05, 'epoch': 0.35}\n",
            "{'eval_stsb_spearman': 0.7948876741884207, 'eval_sickr_spearman': 0.7275022133725474, 'eval_avg_sts': 0.761194943780484, 'epoch': 0.35}\n",
            "{'eval_stsb_spearman': 0.7962851146533464, 'eval_sickr_spearman': 0.7298682254293889, 'eval_avg_sts': 0.7630766700413676, 'epoch': 0.36}\n",
            "{'eval_stsb_spearman': 0.8016911895537082, 'eval_sickr_spearman': 0.7328927919180483, 'eval_avg_sts': 0.7672919907358783, 'epoch': 0.37}\n",
            "{'eval_stsb_spearman': 0.8149143374574255, 'eval_sickr_spearman': 0.7408876167835236, 'eval_avg_sts': 0.7779009771204746, 'epoch': 0.38}\n",
            "{'loss': 0.0001, 'learning_rate': 1.84807372328171e-05, 'epoch': 0.38}\n",
            "{'eval_stsb_spearman': 0.8022292457977832, 'eval_sickr_spearman': 0.7353015516551696, 'eval_avg_sts': 0.7687653987264764, 'epoch': 0.38}\n",
            "{'eval_stsb_spearman': 0.8079200817471521, 'eval_sickr_spearman': 0.7368527641072149, 'eval_avg_sts': 0.7723864229271835, 'epoch': 0.39}\n",
            "{'eval_stsb_spearman': 0.8119458290157565, 'eval_sickr_spearman': 0.7383920168150021, 'eval_avg_sts': 0.7751689229153793, 'epoch': 0.4}\n",
            "{'eval_stsb_spearman': 0.8113324506470739, 'eval_sickr_spearman': 0.739210178596899, 'eval_avg_sts': 0.7752713146219865, 'epoch': 0.41}\n",
            "{'loss': 0.0, 'learning_rate': 1.7520798668885192e-05, 'epoch': 0.42}\n",
            "{'eval_stsb_spearman': 0.8170757862926495, 'eval_sickr_spearman': 0.7384543611846688, 'eval_avg_sts': 0.7777650737386592, 'epoch': 0.42}\n",
            "{'eval_stsb_spearman': 0.8226192466824074, 'eval_sickr_spearman': 0.7410094236567707, 'eval_avg_sts': 0.781814335169589, 'epoch': 0.42}\n",
            "{'eval_stsb_spearman': 0.817365598933365, 'eval_sickr_spearman': 0.7437565625035235, 'eval_avg_sts': 0.7805610807184442, 'epoch': 0.43}\n",
            "{'eval_stsb_spearman': 0.8343981871850513, 'eval_sickr_spearman': 0.7482260005856441, 'eval_avg_sts': 0.7913120938853477, 'epoch': 0.44}\n",
            "{'loss': 0.0002, 'learning_rate': 1.6560860104953284e-05, 'epoch': 0.45}\n",
            "{'eval_stsb_spearman': 0.8266950100128291, 'eval_sickr_spearman': 0.7466562481284439, 'eval_avg_sts': 0.7866756290706365, 'epoch': 0.45}\n",
            "{'eval_stsb_spearman': 0.8256438206752924, 'eval_sickr_spearman': 0.7452526832822193, 'eval_avg_sts': 0.7854482519787558, 'epoch': 0.46}\n",
            "{'eval_stsb_spearman': 0.8246845417001597, 'eval_sickr_spearman': 0.7452181008891838, 'eval_avg_sts': 0.7849513212946717, 'epoch': 0.46}\n",
            "{'eval_stsb_spearman': 0.8116086312741558, 'eval_sickr_spearman': 0.7465727220430429, 'eval_avg_sts': 0.7790906766585994, 'epoch': 0.47}\n",
            "{'loss': 0.0001, 'learning_rate': 1.5600921541021372e-05, 'epoch': 0.48}\n",
            "{'eval_stsb_spearman': 0.801930741320236, 'eval_sickr_spearman': 0.7431478643549982, 'eval_avg_sts': 0.7725393028376171, 'epoch': 0.48}\n",
            "{'eval_stsb_spearman': 0.8039331928783052, 'eval_sickr_spearman': 0.7423119790966709, 'eval_avg_sts': 0.7731225859874881, 'epoch': 0.49}\n",
            "{'eval_stsb_spearman': 0.8147283800338858, 'eval_sickr_spearman': 0.7319622373587863, 'eval_avg_sts': 0.773345308696336, 'epoch': 0.5}\n",
            "{'eval_stsb_spearman': 0.8091585085868589, 'eval_sickr_spearman': 0.7286820493482733, 'eval_avg_sts': 0.7689202789675661, 'epoch': 0.5}\n",
            "{'loss': 0.0001, 'learning_rate': 1.4640982977089467e-05, 'epoch': 0.51}\n",
            "{'eval_stsb_spearman': 0.8086444132072315, 'eval_sickr_spearman': 0.7304633788073085, 'eval_avg_sts': 0.76955389600727, 'epoch': 0.51}\n",
            "{'eval_stsb_spearman': 0.8105341120185496, 'eval_sickr_spearman': 0.7309486370251378, 'eval_avg_sts': 0.7707413745218437, 'epoch': 0.52}\n",
            "{'eval_stsb_spearman': 0.8169400424267292, 'eval_sickr_spearman': 0.7324688694167555, 'eval_avg_sts': 0.7747044559217424, 'epoch': 0.53}\n",
            "{'eval_stsb_spearman': 0.8147490124449251, 'eval_sickr_spearman': 0.7315867782388447, 'eval_avg_sts': 0.7731678953418849, 'epoch': 0.54}\n",
            "{'loss': 0.0001, 'learning_rate': 1.368104441315756e-05, 'epoch': 0.54}\n",
            "{'eval_stsb_spearman': 0.830192981439757, 'eval_sickr_spearman': 0.7249232314119294, 'eval_avg_sts': 0.7775581064258432, 'epoch': 0.54}\n",
            "{'eval_stsb_spearman': 0.824018234144205, 'eval_sickr_spearman': 0.7293347439857157, 'eval_avg_sts': 0.7766764890649605, 'epoch': 0.55}\n",
            "{'eval_stsb_spearman': 0.8194823922072517, 'eval_sickr_spearman': 0.7256114690644362, 'eval_avg_sts': 0.7725469306358439, 'epoch': 0.56}\n",
            "{'eval_stsb_spearman': 0.8320104044005494, 'eval_sickr_spearman': 0.7321967251960072, 'eval_avg_sts': 0.7821035647982784, 'epoch': 0.57}\n",
            "{'loss': 0.0001, 'learning_rate': 1.2721105849225649e-05, 'epoch': 0.58}\n",
            "{'eval_stsb_spearman': 0.8288718239455565, 'eval_sickr_spearman': 0.732038078467957, 'eval_avg_sts': 0.7804549512067567, 'epoch': 0.58}\n",
            "{'eval_stsb_spearman': 0.8275493924291256, 'eval_sickr_spearman': 0.7377952303796336, 'eval_avg_sts': 0.7826723114043797, 'epoch': 0.58}\n",
            "{'eval_stsb_spearman': 0.8268964453598833, 'eval_sickr_spearman': 0.7406971734663215, 'eval_avg_sts': 0.7837968094131024, 'epoch': 0.59}\n",
            "{'eval_stsb_spearman': 0.8299963460729148, 'eval_sickr_spearman': 0.7417935313877478, 'eval_avg_sts': 0.7858949387303313, 'epoch': 0.6}\n",
            "{'loss': 0.0004, 'learning_rate': 1.176116728529374e-05, 'epoch': 0.61}\n",
            "{'eval_stsb_spearman': 0.8315372955335473, 'eval_sickr_spearman': 0.7441766905478027, 'eval_avg_sts': 0.7878569930406749, 'epoch': 0.61}\n",
            "{'eval_stsb_spearman': 0.8273683869662944, 'eval_sickr_spearman': 0.7428192355589587, 'eval_avg_sts': 0.7850938112626266, 'epoch': 0.62}\n",
            "{'eval_stsb_spearman': 0.8259462382992858, 'eval_sickr_spearman': 0.7417601017411469, 'eval_avg_sts': 0.7838531700202164, 'epoch': 0.62}\n",
            "{'eval_stsb_spearman': 0.8264194592660095, 'eval_sickr_spearman': 0.7417379594033838, 'eval_avg_sts': 0.7840787093346966, 'epoch': 0.63}\n",
            "{'loss': 0.0001, 'learning_rate': 1.0801228721361832e-05, 'epoch': 0.64}\n",
            "{'eval_stsb_spearman': 0.8162284796580921, 'eval_sickr_spearman': 0.7458953394194612, 'eval_avg_sts': 0.7810619095387766, 'epoch': 0.64}\n",
            "{'eval_stsb_spearman': 0.8228861792507248, 'eval_sickr_spearman': 0.7415085628629156, 'eval_avg_sts': 0.7821973710568202, 'epoch': 0.65}\n",
            "{'eval_stsb_spearman': 0.8183546537232522, 'eval_sickr_spearman': 0.7410587515979475, 'eval_avg_sts': 0.7797067026605999, 'epoch': 0.66}\n",
            "{'eval_stsb_spearman': 0.8180503937268023, 'eval_sickr_spearman': 0.7408599989001966, 'eval_avg_sts': 0.7794551963134995, 'epoch': 0.66}\n",
            "{'loss': 0.0002, 'learning_rate': 9.841290157429926e-06, 'epoch': 0.67}\n",
            "{'eval_stsb_spearman': 0.8159544403230671, 'eval_sickr_spearman': 0.7387656027219878, 'eval_avg_sts': 0.7773600215225274, 'epoch': 0.67}\n",
            "{'eval_stsb_spearman': 0.8166999264389353, 'eval_sickr_spearman': 0.7391226356379917, 'eval_avg_sts': 0.7779112810384635, 'epoch': 0.68}\n",
            "{'eval_stsb_spearman': 0.8172200116444109, 'eval_sickr_spearman': 0.7390202155907113, 'eval_avg_sts': 0.7781201136175611, 'epoch': 0.69}\n",
            "{'eval_stsb_spearman': 0.8180157648619006, 'eval_sickr_spearman': 0.7389125778923884, 'eval_avg_sts': 0.7784641713771445, 'epoch': 0.7}\n",
            "{'loss': 0.0001, 'learning_rate': 8.881351593498018e-06, 'epoch': 0.7}\n",
            "{'eval_stsb_spearman': 0.816023311531363, 'eval_sickr_spearman': 0.7394554734319434, 'eval_avg_sts': 0.7777393924816531, 'epoch': 0.7}\n",
            "{'eval_stsb_spearman': 0.8142658678175, 'eval_sickr_spearman': 0.7393394302908689, 'eval_avg_sts': 0.7768026490541844, 'epoch': 0.71}\n",
            "{'eval_stsb_spearman': 0.803634394454324, 'eval_sickr_spearman': 0.7350234996089444, 'eval_avg_sts': 0.7693289470316342, 'epoch': 0.72}\n",
            "{'eval_stsb_spearman': 0.8056906371500722, 'eval_sickr_spearman': 0.7354293800664724, 'eval_avg_sts': 0.7705600086082722, 'epoch': 0.73}\n",
            "{'loss': 0.0001, 'learning_rate': 7.921413029566108e-06, 'epoch': 0.74}\n",
            "{'eval_stsb_spearman': 0.8068480205100437, 'eval_sickr_spearman': 0.7352534244915286, 'eval_avg_sts': 0.7710507225007861, 'epoch': 0.74}\n",
            "{'eval_stsb_spearman': 0.8090761857468114, 'eval_sickr_spearman': 0.7349088493698116, 'eval_avg_sts': 0.7719925175583116, 'epoch': 0.74}\n",
            "{'eval_stsb_spearman': 0.8077168680839613, 'eval_sickr_spearman': 0.7331367418822525, 'eval_avg_sts': 0.7704268049831069, 'epoch': 0.75}\n",
            "{'eval_stsb_spearman': 0.8046930001670853, 'eval_sickr_spearman': 0.7336374180836434, 'eval_avg_sts': 0.7691652091253643, 'epoch': 0.76}\n",
            "{'loss': 0.0001, 'learning_rate': 6.961474465634199e-06, 'epoch': 0.77}\n",
            "{'eval_stsb_spearman': 0.8049719871885709, 'eval_sickr_spearman': 0.734028439280451, 'eval_avg_sts': 0.769500213234511, 'epoch': 0.77}\n",
            "{'eval_stsb_spearman': 0.8101048839647094, 'eval_sickr_spearman': 0.7348248429733963, 'eval_avg_sts': 0.7724648634690529, 'epoch': 0.78}\n",
            "{'eval_stsb_spearman': 0.8109162030292262, 'eval_sickr_spearman': 0.734874891381095, 'eval_avg_sts': 0.7728955472051606, 'epoch': 0.78}\n",
            "{'eval_stsb_spearman': 0.8101071704431536, 'eval_sickr_spearman': 0.7349871400651558, 'eval_avg_sts': 0.7725471552541547, 'epoch': 0.79}\n",
            "{'loss': 0.0001, 'learning_rate': 6.001535901702292e-06, 'epoch': 0.8}\n",
            "{'eval_stsb_spearman': 0.8092784136441984, 'eval_sickr_spearman': 0.7339874687509242, 'eval_avg_sts': 0.7716329411975613, 'epoch': 0.8}\n",
            "{'eval_stsb_spearman': 0.809194401130779, 'eval_sickr_spearman': 0.7343813718138184, 'eval_avg_sts': 0.7717878864722987, 'epoch': 0.81}\n",
            "{'eval_stsb_spearman': 0.809642850558119, 'eval_sickr_spearman': 0.7343621113421416, 'eval_avg_sts': 0.7720024809501302, 'epoch': 0.82}\n",
            "{'eval_stsb_spearman': 0.8114967230022869, 'eval_sickr_spearman': 0.73679916139801, 'eval_avg_sts': 0.7741479422001485, 'epoch': 0.82}\n",
            "{'loss': 0.0001, 'learning_rate': 5.0415973377703825e-06, 'epoch': 0.83}\n",
            "{'eval_stsb_spearman': 0.811448833289565, 'eval_sickr_spearman': 0.7366682286154898, 'eval_avg_sts': 0.7740585309525274, 'epoch': 0.83}\n",
            "{'eval_stsb_spearman': 0.8130545763891607, 'eval_sickr_spearman': 0.736211981182929, 'eval_avg_sts': 0.7746332787860448, 'epoch': 0.84}\n",
            "{'eval_stsb_spearman': 0.8126731739910584, 'eval_sickr_spearman': 0.7366621766967085, 'eval_avg_sts': 0.7746676753438835, 'epoch': 0.85}\n",
            "{'eval_stsb_spearman': 0.8097421104130962, 'eval_sickr_spearman': 0.7357737934445091, 'eval_avg_sts': 0.7727579519288026, 'epoch': 0.86}\n",
            "{'loss': 0.0001, 'learning_rate': 4.081658773838474e-06, 'epoch': 0.86}\n",
            "{'eval_stsb_spearman': 0.8074106177009054, 'eval_sickr_spearman': 0.7345794040450477, 'eval_avg_sts': 0.7709950108729766, 'epoch': 0.86}\n",
            "{'eval_stsb_spearman': 0.8078039749055631, 'eval_sickr_spearman': 0.7348392523038279, 'eval_avg_sts': 0.7713216136046954, 'epoch': 0.87}\n",
            "{'eval_stsb_spearman': 0.8104066001955111, 'eval_sickr_spearman': 0.7359770610657953, 'eval_avg_sts': 0.7731918306306532, 'epoch': 0.88}\n",
            "{'eval_stsb_spearman': 0.8112290073466929, 'eval_sickr_spearman': 0.7359132757630854, 'eval_avg_sts': 0.7735711415548892, 'epoch': 0.89}\n",
            "{'loss': 0.0002, 'learning_rate': 3.121720209906566e-06, 'epoch': 0.9}\n",
            "{'eval_stsb_spearman': 0.8113074506298434, 'eval_sickr_spearman': 0.736224805487013, 'eval_avg_sts': 0.7737661280584283, 'epoch': 0.9}\n",
            "{'eval_stsb_spearman': 0.8118547915863824, 'eval_sickr_spearman': 0.73582091195502, 'eval_avg_sts': 0.7738378517707012, 'epoch': 0.9}\n",
            "{'eval_stsb_spearman': 0.8115593696743237, 'eval_sickr_spearman': 0.7351882462868772, 'eval_avg_sts': 0.7733738079806005, 'epoch': 0.91}\n",
            "{'eval_stsb_spearman': 0.8114610450417539, 'eval_sickr_spearman': 0.7353949721474667, 'eval_avg_sts': 0.7734280085946104, 'epoch': 0.92}\n",
            "{'loss': 0.0, 'learning_rate': 2.1617816459746575e-06, 'epoch': 0.93}\n",
            "{'eval_stsb_spearman': 0.8120087536042999, 'eval_sickr_spearman': 0.7358649564750387, 'eval_avg_sts': 0.7739368550396692, 'epoch': 0.93}\n",
            "{'eval_stsb_spearman': 0.8122791843956688, 'eval_sickr_spearman': 0.736151461995117, 'eval_avg_sts': 0.774215323195393, 'epoch': 0.94}\n",
            "{'eval_stsb_spearman': 0.8108153203974181, 'eval_sickr_spearman': 0.7349814243640846, 'eval_avg_sts': 0.7728983723807514, 'epoch': 0.94}\n",
            "{'eval_stsb_spearman': 0.8109274610441028, 'eval_sickr_spearman': 0.7352162484190156, 'eval_avg_sts': 0.7730718547315591, 'epoch': 0.95}\n",
            "{'loss': 0.0001, 'learning_rate': 1.2018430820427492e-06, 'epoch': 0.96}\n",
            "{'eval_stsb_spearman': 0.8110314967029951, 'eval_sickr_spearman': 0.7353644243669522, 'eval_avg_sts': 0.7731979605349737, 'epoch': 0.96}\n",
            "{'eval_stsb_spearman': 0.8117356430957815, 'eval_sickr_spearman': 0.7360765334768736, 'eval_avg_sts': 0.7739060882863276, 'epoch': 0.97}\n",
            "{'eval_stsb_spearman': 0.8116870405809361, 'eval_sickr_spearman': 0.7360993962811581, 'eval_avg_sts': 0.7738932184310471, 'epoch': 0.98}\n",
            "{'eval_stsb_spearman': 0.8117695025497231, 'eval_sickr_spearman': 0.7361710106534024, 'eval_avg_sts': 0.7739702566015627, 'epoch': 0.98}\n",
            "{'loss': 0.0001, 'learning_rate': 2.4190451811084087e-07, 'epoch': 0.99}\n",
            "{'eval_stsb_spearman': 0.811764481915518, 'eval_sickr_spearman': 0.7361939214887884, 'eval_avg_sts': 0.7739792017021532, 'epoch': 0.99}\n",
            "{'eval_stsb_spearman': 0.8117894438446059, 'eval_sickr_spearman': 0.7362039599889889, 'eval_avg_sts': 0.7739967019167975, 'epoch': 1.0}\n",
            "100% 15626/15626 [2:54:38<00:00, 10.38s/it]11/24/2021 00:30:52 - INFO - simcse.trainers -   \n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "11/24/2021 00:30:52 - INFO - simcse.trainers -   Loading best model from result/my-unsup-simcse-bert-base-uncased (score: 0.8344697352770686).\n",
            "[INFO|configuration_utils.py:443] 2021-11-24 00:30:52,009 >> loading configuration file result/my-unsup-simcse-bert-base-uncased/config.json\n",
            "[INFO|configuration_utils.py:481] 2021-11-24 00:30:52,009 >> Model config BertConfig {\n",
            "  \"_name_or_path\": \"bert-base-uncased\",\n",
            "  \"architectures\": [\n",
            "    \"BertForCL\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"gradient_checkpointing\": false,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"layer_norm_eps\": 1e-12,\n",
            "  \"max_position_embeddings\": 512,\n",
            "  \"model_type\": \"bert\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.2.1\",\n",
            "  \"type_vocab_size\": 2,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 30522\n",
            "}\n",
            "\n",
            "[INFO|modeling_utils.py:1025] 2021-11-24 00:30:52,010 >> loading weights file result/my-unsup-simcse-bert-base-uncased/pytorch_model.bin\n",
            "[INFO|modeling_utils.py:1143] 2021-11-24 00:30:55,348 >> All model checkpoint weights were used when initializing BertForCL.\n",
            "\n",
            "[INFO|modeling_utils.py:1152] 2021-11-24 00:30:55,349 >> All the weights of BertForCL were initialized from the model checkpoint at result/my-unsup-simcse-bert-base-uncased.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use BertForCL for predictions without further training.\n",
            "{'train_runtime': 10482.1936, 'train_samples_per_second': 1.491, 'epoch': 1.0}\n",
            "100% 15626/15626 [2:54:42<00:00,  1.49it/s]\n",
            "[INFO|trainer.py:1344] 2021-11-24 00:30:55,572 >> Saving model checkpoint to result/my-unsup-simcse-bert-base-uncased\n",
            "[INFO|configuration_utils.py:300] 2021-11-24 00:30:55,577 >> Configuration saved in result/my-unsup-simcse-bert-base-uncased/config.json\n",
            "[INFO|modeling_utils.py:817] 2021-11-24 00:30:57,513 >> Model weights saved in result/my-unsup-simcse-bert-base-uncased/pytorch_model.bin\n",
            "11/24/2021 00:30:57 - INFO - __main__ -   ***** Train results *****\n",
            "11/24/2021 00:30:57 - INFO - __main__ -     epoch = 1.0\n",
            "11/24/2021 00:30:57 - INFO - __main__ -     train_runtime = 10482.1936\n",
            "11/24/2021 00:30:57 - INFO - __main__ -     train_samples_per_second = 1.491\n",
            "11/24/2021 00:30:57 - INFO - __main__ -   *** Evaluate ***\n",
            "11/24/2021 00:31:32 - INFO - root -   Generating sentence embeddings\n",
            "11/24/2021 00:31:46 - INFO - root -   Generated sentence embeddings\n",
            "11/24/2021 00:31:46 - INFO - root -   Training pytorch-MLP-nhid0-rmsprop-bs128 with (inner) 5-fold cross-validation\n",
            "11/24/2021 00:31:56 - INFO - root -   Best param found at split 1: l2reg = 0.001                 with score 80.55\n",
            "11/24/2021 00:32:09 - INFO - root -   Best param found at split 2: l2reg = 1e-05                 with score 80.88\n",
            "11/24/2021 00:32:21 - INFO - root -   Best param found at split 3: l2reg = 0.001                 with score 81.21\n",
            "11/24/2021 00:32:31 - INFO - root -   Best param found at split 4: l2reg = 0.01                 with score 80.6\n",
            "11/24/2021 00:32:43 - INFO - root -   Best param found at split 5: l2reg = 0.01                 with score 80.43\n",
            "11/24/2021 00:32:44 - INFO - root -   Generating sentence embeddings\n",
            "11/24/2021 00:32:48 - INFO - root -   Generated sentence embeddings\n",
            "11/24/2021 00:32:48 - INFO - root -   Training pytorch-MLP-nhid0-rmsprop-bs128 with (inner) 5-fold cross-validation\n",
            "11/24/2021 00:32:52 - INFO - root -   Best param found at split 1: l2reg = 1e-05                 with score 87.45\n",
            "11/24/2021 00:32:57 - INFO - root -   Best param found at split 2: l2reg = 0.0001                 with score 87.42\n",
            "11/24/2021 00:33:00 - INFO - root -   Best param found at split 3: l2reg = 0.01                 with score 86.32\n",
            "11/24/2021 00:33:04 - INFO - root -   Best param found at split 4: l2reg = 1e-05                 with score 86.56\n",
            "11/24/2021 00:33:08 - INFO - root -   Best param found at split 5: l2reg = 0.0001                 with score 86.62\n",
            "11/24/2021 00:33:09 - INFO - root -   Generating sentence embeddings\n",
            "11/24/2021 00:33:23 - INFO - root -   Generated sentence embeddings\n",
            "11/24/2021 00:33:23 - INFO - root -   Training pytorch-MLP-nhid0-rmsprop-bs128 with (inner) 5-fold cross-validation\n",
            "11/24/2021 00:33:33 - INFO - root -   Best param found at split 1: l2reg = 0.001                 with score 94.71\n",
            "11/24/2021 00:33:45 - INFO - root -   Best param found at split 2: l2reg = 0.001                 with score 94.84\n",
            "11/24/2021 00:33:56 - INFO - root -   Best param found at split 3: l2reg = 1e-05                 with score 94.82\n",
            "11/24/2021 00:34:07 - INFO - root -   Best param found at split 4: l2reg = 0.001                 with score 94.7\n",
            "11/24/2021 00:34:18 - INFO - root -   Best param found at split 5: l2reg = 0.001                 with score 94.58\n",
            "11/24/2021 00:34:20 - INFO - root -   Generating sentence embeddings\n",
            "11/24/2021 00:34:23 - INFO - root -   Generated sentence embeddings\n",
            "11/24/2021 00:34:23 - INFO - root -   Training pytorch-MLP-nhid0-rmsprop-bs128 with (inner) 5-fold cross-validation\n",
            "11/24/2021 00:34:34 - INFO - root -   Best param found at split 1: l2reg = 0.001                 with score 88.86\n",
            "11/24/2021 00:34:44 - INFO - root -   Best param found at split 2: l2reg = 0.01                 with score 87.84\n",
            "11/24/2021 00:34:55 - INFO - root -   Best param found at split 3: l2reg = 0.0001                 with score 88.45\n",
            "11/24/2021 00:35:05 - INFO - root -   Best param found at split 4: l2reg = 0.001                 with score 88.19\n",
            "11/24/2021 00:35:16 - INFO - root -   Best param found at split 5: l2reg = 0.001                 with score 88.77\n",
            "11/24/2021 00:35:19 - INFO - root -   Computing embedding for train\n",
            "11/24/2021 00:36:05 - INFO - root -   Computed train embeddings\n",
            "11/24/2021 00:36:05 - INFO - root -   Computing embedding for dev\n",
            "11/24/2021 00:36:06 - INFO - root -   Computed dev embeddings\n",
            "11/24/2021 00:36:06 - INFO - root -   Computing embedding for test\n",
            "11/24/2021 00:36:08 - INFO - root -   Computed test embeddings\n",
            "11/24/2021 00:36:08 - INFO - root -   Training pytorch-MLP-nhid0-rmsprop-bs128 with standard validation..\n",
            "11/24/2021 00:36:27 - INFO - root -   [('reg:1e-05', 86.47), ('reg:0.0001', 86.35), ('reg:0.001', 85.78), ('reg:0.01', 84.98)]\n",
            "11/24/2021 00:36:27 - INFO - root -   Validation : best param found is reg = 1e-05 with score             86.47\n",
            "11/24/2021 00:36:27 - INFO - root -   Evaluating...\n",
            "11/24/2021 00:36:32 - INFO - root -   ***** Transfer task : TREC *****\n",
            "\n",
            "\n",
            "11/24/2021 00:36:37 - INFO - root -   Computed train embeddings\n",
            "11/24/2021 00:36:37 - INFO - root -   Computed test embeddings\n",
            "11/24/2021 00:36:37 - INFO - root -   Training pytorch-MLP-nhid0-rmsprop-bs128 with 5-fold cross-validation\n",
            "11/24/2021 00:36:44 - INFO - root -   [('reg:1e-05', 80.21), ('reg:0.0001', 80.15), ('reg:0.001', 79.75), ('reg:0.01', 76.78)]\n",
            "11/24/2021 00:36:44 - INFO - root -   Cross-validation : best param found is reg = 1e-05             with score 80.21\n",
            "11/24/2021 00:36:44 - INFO - root -   Evaluating...\n",
            "11/24/2021 00:36:45 - INFO - root -   ***** Transfer task : MRPC *****\n",
            "\n",
            "\n",
            "11/24/2021 00:36:46 - INFO - root -   Computing embedding for train\n",
            "11/24/2021 00:36:56 - INFO - root -   Computed train embeddings\n",
            "11/24/2021 00:36:56 - INFO - root -   Computing embedding for test\n",
            "11/24/2021 00:37:00 - INFO - root -   Computed test embeddings\n",
            "11/24/2021 00:37:00 - INFO - root -   Training pytorch-MLP-nhid0-rmsprop-bs128 with 5-fold cross-validation\n",
            "11/24/2021 00:37:06 - INFO - root -   [('reg:1e-05', 74.73), ('reg:0.0001', 74.78), ('reg:0.001', 74.88), ('reg:0.01', 72.64)]\n",
            "11/24/2021 00:37:06 - INFO - root -   Cross-validation : best param found is reg = 0.001             with score 74.88\n",
            "11/24/2021 00:37:06 - INFO - root -   Evaluating...\n",
            "11/24/2021 00:37:06 - INFO - __main__ -   ***** Eval results *****\n",
            "11/24/2021 00:37:06 - INFO - __main__ -     epoch = 1.0\n",
            "11/24/2021 00:37:06 - INFO - __main__ -     eval_CR = 86.87\n",
            "11/24/2021 00:37:06 - INFO - __main__ -     eval_MPQA = 88.42\n",
            "11/24/2021 00:37:06 - INFO - __main__ -     eval_MR = 80.73\n",
            "11/24/2021 00:37:06 - INFO - __main__ -     eval_MRPC = 74.88\n",
            "11/24/2021 00:37:06 - INFO - __main__ -     eval_SST2 = 86.47\n",
            "11/24/2021 00:37:06 - INFO - __main__ -     eval_SUBJ = 94.73\n",
            "11/24/2021 00:37:06 - INFO - __main__ -     eval_TREC = 80.21\n",
            "11/24/2021 00:37:06 - INFO - __main__ -     eval_avg_sts = 0.7899140675067766\n",
            "11/24/2021 00:37:06 - INFO - __main__ -     eval_avg_transfer = 84.61571428571429\n",
            "11/24/2021 00:37:06 - INFO - __main__ -     eval_sickr_spearman = 0.7453583997364845\n",
            "11/24/2021 00:37:06 - INFO - __main__ -     eval_stsb_spearman = 0.8344697352770686\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uN4hjKhj1nt6",
        "outputId": "96d1f471-689c-409c-bd14-afca22fac97a"
      },
      "source": [
        "!cat run_unsup_example.sh\n",
        "# !wget https://huggingface.co/datasets/princeton-nlp/datasets-for-simcse/resolve/main/wiki1m_for_simcse.txt\n",
        "# !wget https://huggingface.co/datasets/princeton-nlp/datasets-for-simcse/resolve/main/nli_for_simcse.csv"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "cat: run_unsup_example.sh: No such file or directory\n",
            "--2021-11-23 21:05:46--  https://huggingface.co/datasets/princeton-nlp/datasets-for-simcse/resolve/main/wiki1m_for_simcse.txt\n",
            "Resolving huggingface.co (huggingface.co)... 34.204.221.201, 54.221.222.133, 35.173.116.130, ...\n",
            "Connecting to huggingface.co (huggingface.co)|34.204.221.201|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://cdn-lfs.huggingface.co/datasets/princeton-nlp/datasets-for-simcse/7b1825863a99aa76479b0456f7c210539dfaeeb69598b41fb4de4f524dd5a706 [following]\n",
            "--2021-11-23 21:05:47--  https://cdn-lfs.huggingface.co/datasets/princeton-nlp/datasets-for-simcse/7b1825863a99aa76479b0456f7c210539dfaeeb69598b41fb4de4f524dd5a706\n",
            "Resolving cdn-lfs.huggingface.co (cdn-lfs.huggingface.co)... 18.67.65.82, 18.67.65.19, 18.67.65.119, ...\n",
            "Connecting to cdn-lfs.huggingface.co (cdn-lfs.huggingface.co)|18.67.65.82|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 120038621 (114M) [text/plain]\n",
            "Saving to: ‘wiki1m_for_simcse.txt’\n",
            "\n",
            "wiki1m_for_simcse.t 100%[===================>] 114.48M  26.1MB/s    in 4.4s    \n",
            "\n",
            "2021-11-23 21:05:51 (26.2 MB/s) - ‘wiki1m_for_simcse.txt’ saved [120038621/120038621]\n",
            "\n",
            "--2021-11-23 21:05:51--  https://huggingface.co/datasets/princeton-nlp/datasets-for-simcse/resolve/main/nli_for_simcse.csv\n",
            "Resolving huggingface.co (huggingface.co)... 54.221.222.133, 52.22.206.115, 35.173.116.130, ...\n",
            "Connecting to huggingface.co (huggingface.co)|54.221.222.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://cdn-lfs.huggingface.co/datasets/princeton-nlp/datasets-for-simcse/0747687ec3594fa449d2004fd3757a56c24bf5f7428976fb5b67176775a68d48 [following]\n",
            "--2021-11-23 21:05:52--  https://cdn-lfs.huggingface.co/datasets/princeton-nlp/datasets-for-simcse/0747687ec3594fa449d2004fd3757a56c24bf5f7428976fb5b67176775a68d48\n",
            "Resolving cdn-lfs.huggingface.co (cdn-lfs.huggingface.co)... 52.85.79.45, 52.85.79.60, 52.85.79.97, ...\n",
            "Connecting to cdn-lfs.huggingface.co (cdn-lfs.huggingface.co)|52.85.79.45|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 48978197 (47M) [text/plain]\n",
            "Saving to: ‘nli_for_simcse.csv’\n",
            "\n",
            "nli_for_simcse.csv  100%[===================>]  46.71M   106MB/s    in 0.4s    \n",
            "\n",
            "2021-11-23 21:05:52 (106 MB/s) - ‘nli_for_simcse.csv’ saved [48978197/48978197]\n",
            "\n"
          ]
        }
      ]
    }
  ]
}