Task1a.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Task1a.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/RSid8/SMM4H21/blob/main/Task1a.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0NMKmw7EYfVb"
      },
      "source": [
        "#Importing the Libraries and Models"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EleGdQf-TmDq",
        "outputId": "34e35293-3afd-4434-e31a-62a89bed8199"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eb-uZZBWBY-L",
        "outputId": "53e2fb64-40cb-4624-b254-1470090a43a7"
      },
      "source": [
        "!pip install fairseq"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting fairseq\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/15/ab/92c6efb05ffdfe16fbdc9e463229d9af8c3b74dc943ed4b4857a87b223c2/fairseq-0.10.2-cp37-cp37m-manylinux1_x86_64.whl (1.7MB)\n",
            "\u001b[K     |████████████████████████████████| 1.7MB 14.2MB/s \n",
            "\u001b[?25hCollecting hydra-core\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/52/e3/fbd70dd0d3ce4d1d75c22d56c0c9f895cfa7ed6587a9ffb821d6812d6a60/hydra_core-1.0.6-py3-none-any.whl (123kB)\n",
            "\u001b[K     |████████████████████████████████| 133kB 54.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: torch in /usr/local/lib/python3.7/dist-packages (from fairseq) (1.8.0+cu101)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from fairseq) (1.19.5)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from fairseq) (4.41.1)\n",
            "Collecting sacrebleu>=1.4.12\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7e/57/0c7ca4e31a126189dab99c19951910bd081dea5bbd25f24b77107750eae7/sacrebleu-1.5.1-py3-none-any.whl (54kB)\n",
            "\u001b[K     |████████████████████████████████| 61kB 10.0MB/s \n",
            "\u001b[?25hCollecting dataclasses\n",
            "  Downloading https://files.pythonhosted.org/packages/26/2f/1095cdc2868052dd1e64520f7c0d5c8c550ad297e944e641dbf1ffbb9a5d/dataclasses-0.6-py3-none-any.whl\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.7/dist-packages (from fairseq) (2019.12.20)\n",
            "Requirement already satisfied: cffi in /usr/local/lib/python3.7/dist-packages (from fairseq) (1.14.5)\n",
            "Requirement already satisfied: cython in /usr/local/lib/python3.7/dist-packages (from fairseq) (0.29.22)\n",
            "Collecting omegaconf<2.1,>=2.0.5\n",
            "  Downloading https://files.pythonhosted.org/packages/d0/eb/9d63ce09dd8aa85767c65668d5414958ea29648a0eec80a4a7d311ec2684/omegaconf-2.0.6-py3-none-any.whl\n",
            "Collecting antlr4-python3-runtime==4.8\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/56/02/789a0bddf9c9b31b14c3e79ec22b9656185a803dc31c15f006f9855ece0d/antlr4-python3-runtime-4.8.tar.gz (112kB)\n",
            "\u001b[K     |████████████████████████████████| 112kB 56.3MB/s \n",
            "\u001b[?25hRequirement already satisfied: importlib-resources; python_version < \"3.9\" in /usr/local/lib/python3.7/dist-packages (from hydra-core->fairseq) (5.1.2)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch->fairseq) (3.7.4.3)\n",
            "Collecting portalocker==2.0.0\n",
            "  Downloading https://files.pythonhosted.org/packages/89/a6/3814b7107e0788040870e8825eebf214d72166adf656ba7d4bf14759a06a/portalocker-2.0.0-py2.py3-none-any.whl\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.7/dist-packages (from cffi->fairseq) (2.20)\n",
            "Collecting PyYAML>=5.1.*\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7a/a5/393c087efdc78091afa2af9f1378762f9821c9c1d7a22c5753fb5ac5f97a/PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl (636kB)\n",
            "\u001b[K     |████████████████████████████████| 645kB 57.7MB/s \n",
            "\u001b[?25hRequirement already satisfied: zipp>=0.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-resources; python_version < \"3.9\"->hydra-core->fairseq) (3.4.1)\n",
            "Building wheels for collected packages: antlr4-python3-runtime\n",
            "  Building wheel for antlr4-python3-runtime (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for antlr4-python3-runtime: filename=antlr4_python3_runtime-4.8-cp37-none-any.whl size=141231 sha256=f53e595e77b60c22b734ec8ed730b5b5dddae40e74f302ed5ec12d1f9eb68701\n",
            "  Stored in directory: /root/.cache/pip/wheels/e3/e2/fa/b78480b448b8579ddf393bebd3f47ee23aa84c89b6a78285c8\n",
            "Successfully built antlr4-python3-runtime\n",
            "Installing collected packages: PyYAML, omegaconf, antlr4-python3-runtime, hydra-core, portalocker, sacrebleu, dataclasses, fairseq\n",
            "  Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "Successfully installed PyYAML-5.4.1 antlr4-python3-runtime-4.8 dataclasses-0.6 fairseq-0.10.2 hydra-core-1.0.6 omegaconf-2.0.6 portalocker-2.0.0 sacrebleu-1.5.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6zkXHXDi1zj_",
        "outputId": "a30cd80d-502e-4e02-9a5e-110638d37542"
      },
      "source": [
        "!git clone https://github.com/pytorch/fairseq\n",
        "%cd fairseq"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'fairseq'...\n",
            "remote: Enumerating objects: 55, done.\u001b[K\n",
            "remote: Counting objects: 100% (55/55), done.\u001b[K\n",
            "remote: Compressing objects: 100% (36/36), done.\u001b[K\n",
            "remote: Total 21750 (delta 21), reused 26 (delta 16), pack-reused 21695\u001b[K\n",
            "Receiving objects: 100% (21750/21750), 10.08 MiB | 21.96 MiB/s, done.\n",
            "Resolving deltas: 100% (16218/16218), done.\n",
            "/content/fairseq\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wcPvqInp64Fs",
        "outputId": "a434f7f1-3583-4a82-a345-21f1a1037da3"
      },
      "source": [
        "%%shell\n",
        "wget https://dl.fbaipublicfiles.com/fairseq/models/roberta.large.tar.gz\n",
        "tar -xzvf roberta.large.tar.gz"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2021-03-10 06:52:22--  https://dl.fbaipublicfiles.com/fairseq/models/roberta.large.tar.gz\n",
            "Resolving dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)... 104.22.74.142, 172.67.9.4, 104.22.75.142, ...\n",
            "Connecting to dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)|104.22.74.142|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 655283069 (625M) [application/gzip]\n",
            "Saving to: ‘roberta.large.tar.gz’\n",
            "\n",
            "roberta.large.tar.g 100%[===================>] 624.93M  12.2MB/s    in 53s     \n",
            "\n",
            "2021-03-10 06:53:15 (11.9 MB/s) - ‘roberta.large.tar.gz’ saved [655283069/655283069]\n",
            "\n",
            "roberta.large/\n",
            "roberta.large/dict.txt\n",
            "roberta.large/model.pt\n",
            "roberta.large/NOTE\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              ""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ylYe3wFK3VQl",
        "outputId": "dbf3f366-20c7-4a0d-cc1c-3efac287a241"
      },
      "source": [
        "import torch\n",
        "from tqdm import tqdm\n",
        "roberta = torch.hub.load('pytorch/fairseq', 'roberta.large')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading: \"https://github.com/pytorch/fairseq/archive/master.zip\" to /root/.cache/torch/hub/master.zip\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "running build_ext\n",
            "cythoning fairseq/data/data_utils_fast.pyx to fairseq/data/data_utils_fast.cpp\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/utils/cpp_extension.py:369: UserWarning: Attempted to use ninja as the BuildExtension backend but we could not find ninja.. Falling back to using the slow distutils backend.\n",
            "  warnings.warn(msg.format('we could not find ninja.'))\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "cythoning fairseq/data/token_block_utils_fast.pyx to fairseq/data/token_block_utils_fast.cpp\n",
            "building 'fairseq.libbleu' extension\n",
            "creating build\n",
            "creating build/temp.linux-x86_64-3.7\n",
            "creating build/temp.linux-x86_64-3.7/fairseq\n",
            "creating build/temp.linux-x86_64-3.7/fairseq/clib\n",
            "creating build/temp.linux-x86_64-3.7/fairseq/clib/libbleu\n",
            "x86_64-linux-gnu-gcc -pthread -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -O2 -Wall -g -fdebug-prefix-map=/build/python3.7-a56wZI/python3.7-3.7.10=. -fstack-protector-strong -Wformat -Werror=format-security -g -fdebug-prefix-map=/build/python3.7-a56wZI/python3.7-3.7.10=. -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I/usr/include/python3.7m -c fairseq/clib/libbleu/libbleu.cpp -o build/temp.linux-x86_64-3.7/fairseq/clib/libbleu/libbleu.o -std=c++11 -O3 -DTORCH_API_INCLUDE_EXTENSION_H -DPYBIND11_COMPILER_TYPE=\"_gcc\" -DPYBIND11_STDLIB=\"_libstdcpp\" -DPYBIND11_BUILD_ABI=\"_cxxabi1011\" -DTORCH_EXTENSION_NAME=libbleu -D_GLIBCXX_USE_CXX11_ABI=0\n",
            "x86_64-linux-gnu-gcc -pthread -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -O2 -Wall -g -fdebug-prefix-map=/build/python3.7-a56wZI/python3.7-3.7.10=. -fstack-protector-strong -Wformat -Werror=format-security -g -fdebug-prefix-map=/build/python3.7-a56wZI/python3.7-3.7.10=. -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I/usr/include/python3.7m -c fairseq/clib/libbleu/module.cpp -o build/temp.linux-x86_64-3.7/fairseq/clib/libbleu/module.o -std=c++11 -O3 -DTORCH_API_INCLUDE_EXTENSION_H -DPYBIND11_COMPILER_TYPE=\"_gcc\" -DPYBIND11_STDLIB=\"_libstdcpp\" -DPYBIND11_BUILD_ABI=\"_cxxabi1011\" -DTORCH_EXTENSION_NAME=libbleu -D_GLIBCXX_USE_CXX11_ABI=0\n",
            "creating build/lib.linux-x86_64-3.7\n",
            "creating build/lib.linux-x86_64-3.7/fairseq\n",
            "x86_64-linux-gnu-g++ -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-Bsymbolic-functions -Wl,-z,relro -Wl,-Bsymbolic-functions -Wl,-z,relro -g -fdebug-prefix-map=/build/python3.7-a56wZI/python3.7-3.7.10=. -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 build/temp.linux-x86_64-3.7/fairseq/clib/libbleu/libbleu.o build/temp.linux-x86_64-3.7/fairseq/clib/libbleu/module.o -o build/lib.linux-x86_64-3.7/fairseq/libbleu.cpython-37m-x86_64-linux-gnu.so\n",
            "building 'fairseq.data.data_utils_fast' extension\n",
            "creating build/temp.linux-x86_64-3.7/fairseq/data\n",
            "x86_64-linux-gnu-gcc -pthread -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -O2 -Wall -g -fdebug-prefix-map=/build/python3.7-a56wZI/python3.7-3.7.10=. -fstack-protector-strong -Wformat -Werror=format-security -g -fdebug-prefix-map=/build/python3.7-a56wZI/python3.7-3.7.10=. -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I/usr/local/lib/python3.7/dist-packages/numpy/core/include -I/usr/local/lib/python3.7/dist-packages/numpy/core/include -I/usr/include/python3.7m -c fairseq/data/data_utils_fast.cpp -o build/temp.linux-x86_64-3.7/fairseq/data/data_utils_fast.o -std=c++11 -O3 -DTORCH_API_INCLUDE_EXTENSION_H -DPYBIND11_COMPILER_TYPE=\"_gcc\" -DPYBIND11_STDLIB=\"_libstdcpp\" -DPYBIND11_BUILD_ABI=\"_cxxabi1011\" -DTORCH_EXTENSION_NAME=data_utils_fast -D_GLIBCXX_USE_CXX11_ABI=0\n",
            "creating build/lib.linux-x86_64-3.7/fairseq/data\n",
            "x86_64-linux-gnu-g++ -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-Bsymbolic-functions -Wl,-z,relro -Wl,-Bsymbolic-functions -Wl,-z,relro -g -fdebug-prefix-map=/build/python3.7-a56wZI/python3.7-3.7.10=. -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 build/temp.linux-x86_64-3.7/fairseq/data/data_utils_fast.o -o build/lib.linux-x86_64-3.7/fairseq/data/data_utils_fast.cpython-37m-x86_64-linux-gnu.so\n",
            "building 'fairseq.data.token_block_utils_fast' extension\n",
            "x86_64-linux-gnu-gcc -pthread -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -O2 -Wall -g -fdebug-prefix-map=/build/python3.7-a56wZI/python3.7-3.7.10=. -fstack-protector-strong -Wformat -Werror=format-security -g -fdebug-prefix-map=/build/python3.7-a56wZI/python3.7-3.7.10=. -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I/usr/local/lib/python3.7/dist-packages/numpy/core/include -I/usr/local/lib/python3.7/dist-packages/numpy/core/include -I/usr/include/python3.7m -c fairseq/data/token_block_utils_fast.cpp -o build/temp.linux-x86_64-3.7/fairseq/data/token_block_utils_fast.o -std=c++11 -O3 -DTORCH_API_INCLUDE_EXTENSION_H -DPYBIND11_COMPILER_TYPE=\"_gcc\" -DPYBIND11_STDLIB=\"_libstdcpp\" -DPYBIND11_BUILD_ABI=\"_cxxabi1011\" -DTORCH_EXTENSION_NAME=token_block_utils_fast -D_GLIBCXX_USE_CXX11_ABI=0\n",
            "x86_64-linux-gnu-g++ -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-Bsymbolic-functions -Wl,-z,relro -Wl,-Bsymbolic-functions -Wl,-z,relro -g -fdebug-prefix-map=/build/python3.7-a56wZI/python3.7-3.7.10=. -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 build/temp.linux-x86_64-3.7/fairseq/data/token_block_utils_fast.o -o build/lib.linux-x86_64-3.7/fairseq/data/token_block_utils_fast.cpython-37m-x86_64-linux-gnu.so\n",
            "building 'fairseq.libnat' extension\n",
            "creating build/temp.linux-x86_64-3.7/fairseq/clib/libnat\n",
            "x86_64-linux-gnu-gcc -pthread -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -O2 -Wall -g -fdebug-prefix-map=/build/python3.7-a56wZI/python3.7-3.7.10=. -fstack-protector-strong -Wformat -Werror=format-security -g -fdebug-prefix-map=/build/python3.7-a56wZI/python3.7-3.7.10=. -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I/usr/local/lib/python3.7/dist-packages/torch/include -I/usr/local/lib/python3.7/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.7/dist-packages/torch/include/TH -I/usr/local/lib/python3.7/dist-packages/torch/include/THC -I/usr/include/python3.7m -c fairseq/clib/libnat/edit_dist.cpp -o build/temp.linux-x86_64-3.7/fairseq/clib/libnat/edit_dist.o -DTORCH_API_INCLUDE_EXTENSION_H -DPYBIND11_COMPILER_TYPE=\"_gcc\" -DPYBIND11_STDLIB=\"_libstdcpp\" -DPYBIND11_BUILD_ABI=\"_cxxabi1011\" -DTORCH_EXTENSION_NAME=libnat -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++14\n",
            "x86_64-linux-gnu-g++ -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-Bsymbolic-functions -Wl,-z,relro -Wl,-Bsymbolic-functions -Wl,-z,relro -g -fdebug-prefix-map=/build/python3.7-a56wZI/python3.7-3.7.10=. -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 build/temp.linux-x86_64-3.7/fairseq/clib/libnat/edit_dist.o -L/usr/local/lib/python3.7/dist-packages/torch/lib -lc10 -ltorch -ltorch_cpu -ltorch_python -o build/lib.linux-x86_64-3.7/fairseq/libnat.cpython-37m-x86_64-linux-gnu.so\n",
            "copying build/lib.linux-x86_64-3.7/fairseq/libbleu.cpython-37m-x86_64-linux-gnu.so -> fairseq\n",
            "copying build/lib.linux-x86_64-3.7/fairseq/data/data_utils_fast.cpython-37m-x86_64-linux-gnu.so -> fairseq/data\n",
            "copying build/lib.linux-x86_64-3.7/fairseq/data/token_block_utils_fast.cpython-37m-x86_64-linux-gnu.so -> fairseq/data\n",
            "copying build/lib.linux-x86_64-3.7/fairseq/libnat.cpython-37m-x86_64-linux-gnu.so -> fairseq\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 655283069/655283069 [00:54<00:00, 12079007.59B/s]\n",
            "1042301B [00:00, 1193226.37B/s]\n",
            "456318B [00:00, 668520.61B/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ASEHrGOA7KrO",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e1f07e16-a1f2-4365-e146-1f8d65984e88"
      },
      "source": [
        "!ls"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "CODE_OF_CONDUCT.md  fairseq\t pyproject.toml        scripts\n",
            "CONTRIBUTING.md     fairseq_cli  README.md\t       setup.py\n",
            "docs\t\t    hubconf.py\t roberta.large\t       tests\n",
            "examples\t    LICENSE\t roberta.large.tar.gz  train.py\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "qTJNIsQp426z",
        "outputId": "491ba57f-dbc7-45b8-97cc-b90b80c509e0"
      },
      "source": [
        "tokens = roberta.encode('Hello world!')\n",
        "assert tokens.tolist() == [0, 31414, 232, 328, 2]\n",
        "roberta.decode(tokens)  # 'Hello world!'"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'Hello world!'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "toYTr020YrLF"
      },
      "source": [
        "# Preprocessing the data for training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "OpGTydci5GAQ",
        "outputId": "d76cce96-f28c-4775-c354-396ccff96cac"
      },
      "source": [
        "import pandas as pd\n",
        "df = pd.read_csv(\"/content/drive/MyDrive/UPENN/Task1a/train.tsv\", sep = '\\t')\n",
        "df_tweets = pd.read_csv('/content/drive/MyDrive/UPENN/Task1a/tweets.tsv', sep = '\\t')\n",
        "df_class = pd.read_csv('/content/drive/MyDrive/UPENN/Task1a/class.tsv', sep = '\\t')\n",
        "df.columns = [\"tweet_id\", \"tweet\", \"label\"]\n",
        "df_valid = pd.merge(df_tweets, df_class, on = 'tweet_id')\n",
        "df = pd.concat([df, df_valid], axis=0)\n",
        "df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tweet_id</th>\n",
              "      <th>tweet</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>349415675556667392</td>\n",
              "      <td>methylpred, glatiramer acetate, interferon alp...</td>\n",
              "      <td>NoADE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>342091528799395840</td>\n",
              "      <td>@fmab_pride @crazykidwrath_ // .... cymbalta c...</td>\n",
              "      <td>NoADE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>536298734997741569</td>\n",
              "      <td>@rossbalham I don't think Imodium works . Full...</td>\n",
              "      <td>NoADE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>326749581171912704</td>\n",
              "      <td>@EvilGalProds Meanwhile, all I get is flavorle...</td>\n",
              "      <td>NoADE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>536113375085068288</td>\n",
              "      <td>Does Imodium treat fat tummy?</td>\n",
              "      <td>NoADE</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "             tweet_id                                              tweet  label\n",
              "0  349415675556667392  methylpred, glatiramer acetate, interferon alp...  NoADE\n",
              "1  342091528799395840  @fmab_pride @crazykidwrath_ // .... cymbalta c...  NoADE\n",
              "2  536298734997741569  @rossbalham I don't think Imodium works . Full...  NoADE\n",
              "3  326749581171912704  @EvilGalProds Meanwhile, all I get is flavorle...  NoADE\n",
              "4  536113375085068288                      Does Imodium treat fat tummy?  NoADE"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8PXEse2Hqy8e",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "623af198-66b6-463b-d851-ce2c07f3b99b"
      },
      "source": [
        "df.label.value_counts()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "NoADE    16959\n",
              "ADE       1297\n",
              "Name: label, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ts-xijTBq43W",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5f201d81-48d7-493d-b0aa-b8acc7dcf310"
      },
      "source": [
        "df.tweet_id.nunique()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "18256"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X24GSZQkQp3X",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "25713229-d2d1-4ad0-c109-d73d8a8892a3"
      },
      "source": [
        "import numpy as np\n",
        "# count = df['tweet'].str.split().apply(len).value_counts()\n",
        "# count.index = count.index.astype(str) + ' words:'\n",
        "# count.sort_index(inplace=True)\n",
        "# count\n",
        "a = np.array(df['tweet'].str.split().apply(len))\n",
        "print(f'Longest sentence {a.max()}, smallest sentence {a.min()}, average sentence length {a.mean()}')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Longest sentence 443, smallest sentence 1, average sentence length 16.054064417177916\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HjbYFuX_BT3t",
        "outputId": "f2eca994-0a9a-4b65-9a4a-0b3d6eb4af23"
      },
      "source": [
        "# something is wrong in example - 11986"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "303    24\n",
            "303    12\n",
            "443     8\n",
            "443    26\n",
            "Name: tweet, dtype: int64\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8EC4sEi8TgAE",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e91f8fc8-0385-4db8-a35b-9af7092910b8"
      },
      "source": [
        "index_names = df[df['tweet'].str.split().apply(len)>35].index\n",
        "df.drop(index_names, inplace=True)\n",
        "df.tweet_id.nunique()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "18251"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mkrU2RpNhu0j",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "outputId": "fcb7f401-b078-4f2a-bdde-f4a27a6c910a"
      },
      "source": [
        "df['label'].replace({\"NoADE\":0, \"ADE\":1}, inplace=True)\n",
        "df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tweet_id</th>\n",
              "      <th>tweet</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>349415675556667392</td>\n",
              "      <td>methylpred, glatiramer acetate, interferon alp...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>342091528799395840</td>\n",
              "      <td>@fmab_pride @crazykidwrath_ // .... cymbalta c...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>536298734997741569</td>\n",
              "      <td>@rossbalham I don't think Imodium works . Full...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>326749581171912704</td>\n",
              "      <td>@EvilGalProds Meanwhile, all I get is flavorle...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>536113375085068288</td>\n",
              "      <td>Does Imodium treat fat tummy?</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "             tweet_id                                              tweet  label\n",
              "0  349415675556667392  methylpred, glatiramer acetate, interferon alp...      0\n",
              "1  342091528799395840  @fmab_pride @crazykidwrath_ // .... cymbalta c...      0\n",
              "2  536298734997741569  @rossbalham I don't think Imodium works . Full...      0\n",
              "3  326749581171912704  @EvilGalProds Meanwhile, all I get is flavorle...      0\n",
              "4  536113375085068288                      Does Imodium treat fat tummy?      0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e2p62HoVy-uT",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "801d78be-8d9a-49df-94a6-a22c2596fd7d"
      },
      "source": [
        "import os\n",
        "import random\n",
        "from glob import glob\n",
        "import sklearn\n",
        "from sklearn.model_selection import train_test_split\n",
        "from collections import Counter\n",
        "from imblearn.over_sampling import RandomOverSampler\n",
        "from imblearn.under_sampling import RandomUnderSampler\n",
        "\n",
        "X_train,X_val, Y_train, Y_val= train_test_split(df['tweet'], df['label'], test_size = 0.1, random_state = 21)\n",
        "X_train.reset_index(drop=True, inplace = True)\n",
        "X_val.reset_index(drop=True, inplace = True)\n",
        "Y_train.reset_index(drop=True, inplace = True)\n",
        "Y_val.reset_index(drop=True, inplace=True)\n",
        "# define oversampling strategy\n",
        "over = RandomOverSampler(sampling_strategy=0.1)\n",
        "# define undersampling strategy\n",
        "under = RandomUnderSampler(sampling_strategy=0.5)\n",
        "X_train = X_train.values.reshape(-1, 1)\n",
        "X_train, Y_train = over.fit_resample(X_train, Y_train)\n",
        "X_train, Y_train = under.fit_resample(X_train, Y_train)\n",
        "print(Counter(Y_train))\n",
        "print(X_train[0][0])\n",
        "for split in ['train', 'val']:\n",
        "  out_fname = 'train' if split == 'train' else 'val'\n",
        "  f1 = open(os.path.join(\"/content/drive/MyDrive/UPENN/Task1a\", out_fname+'.input0'), 'w')\n",
        "  f2 = open(os.path.join(\"/content/drive/MyDrive/UPENN/Task1a\", out_fname+'.label'), 'w')\n",
        "  if split=='train':\n",
        "    for i in range(len(X_train)):\n",
        "      f1.write(str(X_train[i][0])+'\\n')\n",
        "      f2.write(str(Y_train[i])+'\\n')\n",
        "  else:\n",
        "    for i in range(len(X_val)):\n",
        "      f1.write(X_val[i]+'\\n')\n",
        "      f2.write(str(Y_val[i])+'\\n')\n",
        "  f1.close()\n",
        "  f2.close()\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/externals/six.py:31: FutureWarning: The module is deprecated in version 0.21 and will be removed in version 0.23 since we've dropped support for Python 2.7. Please rely on the official version of six (https://pypi.org/project/six/).\n",
            "  \"(https://pypi.org/project/six/).\", FutureWarning)\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/deprecation.py:144: FutureWarning: The sklearn.neighbors.base module is  deprecated in version 0.22 and will be removed in version 0.24. The corresponding classes / functions should instead be imported from sklearn.neighbors. Anything that cannot be imported from sklearn.neighbors is now part of the private API.\n",
            "  warnings.warn(message, FutureWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Counter({0: 3048, 1: 1524})\n",
            "Average weight gain in 12 weeks of Rx for olanzapine in kids - 8.5kg!! And even Abilify - 5.4kg!! #CON13\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n",
            "  warnings.warn(msg, category=FutureWarning)\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.\n",
            "  warnings.warn(msg, category=FutureWarning)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CacWiBLWYOpG"
      },
      "source": [
        "# Tokenize the data and Finetune Roberta"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NmUvqDWjmshG",
        "outputId": "b1ad93f5-c8d2-4833-d10d-3eb0d914c0aa"
      },
      "source": [
        "%%shell\n",
        "wget -N 'https://dl.fbaipublicfiles.com/fairseq/gpt2_bpe/encoder.json'\n",
        "wget -N 'https://dl.fbaipublicfiles.com/fairseq/gpt2_bpe/vocab.bpe'\n",
        "\n",
        "for SPLIT in train val; do\n",
        "    python -m examples.roberta.multiprocessing_bpe_encoder \\\n",
        "        --encoder-json encoder.json \\\n",
        "        --vocab-bpe vocab.bpe \\\n",
        "        --inputs \"/content/drive/MyDrive/UPENN/Task1a/$SPLIT.input0\" \\\n",
        "        --outputs \"/content/drive/MyDrive/UPENN/Task1a/$SPLIT.input0.bpe\" \\\n",
        "        --workers 60 \\\n",
        "        --keep-empty\n",
        "done"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2021-02-28 16:29:06--  https://dl.fbaipublicfiles.com/fairseq/gpt2_bpe/encoder.json\n",
            "Resolving dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)... 104.22.75.142, 172.67.9.4, 104.22.74.142, ...\n",
            "Connecting to dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)|104.22.75.142|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 1042301 (1018K) [text/plain]\n",
            "Saving to: ‘encoder.json’\n",
            "\n",
            "encoder.json        100%[===================>]   1018K  1.33MB/s    in 0.7s    \n",
            "\n",
            "2021-02-28 16:29:07 (1.33 MB/s) - ‘encoder.json’ saved [1042301/1042301]\n",
            "\n",
            "--2021-02-28 16:29:07--  https://dl.fbaipublicfiles.com/fairseq/gpt2_bpe/vocab.bpe\n",
            "Resolving dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)... 104.22.75.142, 172.67.9.4, 104.22.74.142, ...\n",
            "Connecting to dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)|104.22.75.142|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 456318 (446K) [text/plain]\n",
            "Saving to: ‘vocab.bpe’\n",
            "\n",
            "vocab.bpe           100%[===================>] 445.62K   756KB/s    in 0.6s    \n",
            "\n",
            "2021-02-28 16:29:08 (756 KB/s) - ‘vocab.bpe’ saved [456318/456318]\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              ""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1m7Pt1sf3Ah0",
        "outputId": "fe55ac30-f9be-4a17-f53d-a9f478255ddd"
      },
      "source": [
        "%%shell\n",
        "wget -N 'https://dl.fbaipublicfiles.com/fairseq/gpt2_bpe/dict.txt'  \n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2021-02-28 16:29:34--  https://dl.fbaipublicfiles.com/fairseq/gpt2_bpe/dict.txt\n",
            "Resolving dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)... 104.22.75.142, 172.67.9.4, 104.22.74.142, ...\n",
            "Connecting to dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)|104.22.75.142|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 603290 (589K) [text/plain]\n",
            "Saving to: ‘dict.txt’\n",
            "\n",
            "dict.txt            100%[===================>] 589.15K   990KB/s    in 0.6s    \n",
            "\n",
            "2021-02-28 16:29:36 (990 KB/s) - ‘dict.txt’ saved [603290/603290]\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              ""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VsH8UWJ4Pq-x",
        "outputId": "91776c73-39c0-4d64-db4f-deacc3192be5"
      },
      "source": [
        "%%bash\n",
        "fairseq-preprocess \\\n",
        "    --only-source \\\n",
        "    --trainpref \"/content/drive/MyDrive/UPENN/Task1a/train.input0.bpe\" \\\n",
        "    --validpref \"/content/drive/MyDrive/UPENN/Task1a/val.input0.bpe\" \\\n",
        "    --destdir \"/content/drive/MyDrive/UPENN/Task1a-bin/input0\" \\\n",
        "    --workers 60 \\\n",
        "    --srcdict dict.txt\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-02-28 16:29:38 | INFO | fairseq_cli.preprocess | Namespace(align_suffix=None, alignfile=None, all_gather_list_size=16384, bf16=False, bpe=None, checkpoint_shard_count=1, checkpoint_suffix='', cpu=False, criterion='cross_entropy', dataset_impl='mmap', destdir='/content/drive/MyDrive/UPENN/Task1a-bin/input0', empty_cache_freq=0, fp16=False, fp16_init_scale=128, fp16_no_flatten_grads=False, fp16_scale_tolerance=0.0, fp16_scale_window=None, joined_dictionary=False, log_format=None, log_interval=100, lr_scheduler='fixed', memory_efficient_bf16=False, memory_efficient_fp16=False, min_loss_scale=0.0001, model_parallel_size=1, no_progress_bar=False, nwordssrc=-1, nwordstgt=-1, only_source=True, optimizer=None, padding_factor=8, profile=False, quantization_config_path=None, scoring='bleu', seed=1, source_lang=None, srcdict='dict.txt', target_lang=None, task='translation', tensorboard_logdir=None, testpref=None, tgtdict=None, threshold_loss_scale=None, thresholdsrc=0, thresholdtgt=0, tokenizer=None, tpu=False, trainpref='/content/drive/MyDrive/UPENN/Task1a/train.input0.bpe', user_dir=None, validpref='/content/drive/MyDrive/UPENN/Task1a/val.input0.bpe', workers=60)\n",
            "2021-02-28 16:29:38 | INFO | fairseq_cli.preprocess | [None] Dictionary: 50264 types\n",
            "2021-02-28 16:29:44 | INFO | fairseq_cli.preprocess | [None] /content/drive/MyDrive/UPENN/Task1a/train.input0.bpe: 4572 sents, 132354 tokens, 0.0% replaced by <unk>\n",
            "2021-02-28 16:29:44 | INFO | fairseq_cli.preprocess | [None] Dictionary: 50264 types\n",
            "2021-02-28 16:29:48 | INFO | fairseq_cli.preprocess | [None] /content/drive/MyDrive/UPENN/Task1a/val.input0.bpe: 1826 sents, 51455 tokens, 0.0% replaced by <unk>\n",
            "2021-02-28 16:29:48 | INFO | fairseq_cli.preprocess | Wrote preprocessed data to /content/drive/MyDrive/UPENN/Task1a-bin/input0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nmyWXic2PsxU",
        "outputId": "73455167-1bd5-408d-ef69-8aa64266114d"
      },
      "source": [
        "%%bash\n",
        "fairseq-preprocess \\\n",
        "    --only-source \\\n",
        "    --trainpref \"/content/drive/MyDrive/UPENN/Task1a/train.label\" \\\n",
        "    --validpref \"/content/drive/MyDrive/UPENN/Task1a/val.label\" \\\n",
        "    --destdir \"/content/drive/MyDrive/UPENN/Task1a-bin/label\" \\\n",
        "    --workers 60"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-02-28 16:29:49 | INFO | fairseq_cli.preprocess | Namespace(align_suffix=None, alignfile=None, all_gather_list_size=16384, bf16=False, bpe=None, checkpoint_shard_count=1, checkpoint_suffix='', cpu=False, criterion='cross_entropy', dataset_impl='mmap', destdir='/content/drive/MyDrive/UPENN/Task1a-bin/label', empty_cache_freq=0, fp16=False, fp16_init_scale=128, fp16_no_flatten_grads=False, fp16_scale_tolerance=0.0, fp16_scale_window=None, joined_dictionary=False, log_format=None, log_interval=100, lr_scheduler='fixed', memory_efficient_bf16=False, memory_efficient_fp16=False, min_loss_scale=0.0001, model_parallel_size=1, no_progress_bar=False, nwordssrc=-1, nwordstgt=-1, only_source=True, optimizer=None, padding_factor=8, profile=False, quantization_config_path=None, scoring='bleu', seed=1, source_lang=None, srcdict=None, target_lang=None, task='translation', tensorboard_logdir=None, testpref=None, tgtdict=None, threshold_loss_scale=None, thresholdsrc=0, thresholdtgt=0, tokenizer=None, tpu=False, trainpref='/content/drive/MyDrive/UPENN/Task1a/train.label', user_dir=None, validpref='/content/drive/MyDrive/UPENN/Task1a/val.label', workers=60)\n",
            "2021-02-28 16:29:50 | INFO | fairseq_cli.preprocess | [None] Dictionary: 8 types\n",
            "2021-02-28 16:29:53 | INFO | fairseq_cli.preprocess | [None] /content/drive/MyDrive/UPENN/Task1a/train.label: 4572 sents, 9144 tokens, 0.0% replaced by <unk>\n",
            "2021-02-28 16:29:53 | INFO | fairseq_cli.preprocess | [None] Dictionary: 8 types\n",
            "2021-02-28 16:29:55 | INFO | fairseq_cli.preprocess | [None] /content/drive/MyDrive/UPENN/Task1a/val.label: 1826 sents, 3652 tokens, 0.0% replaced by <unk>\n",
            "2021-02-28 16:29:55 | INFO | fairseq_cli.preprocess | Wrote preprocessed data to /content/drive/MyDrive/UPENN/Task1a-bin/label\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qNJeHBirn3QJ",
        "outputId": "cbada2b9-83fa-48a0-8bd1-a984d3e4e8f8"
      },
      "source": [
        "%%shell\n",
        "TOTAL_NUM_UPDATES=3614  # 10 epochs through UPENN for bsz 32\n",
        "WARMUP_UPDATES=217      # 6 percent of the number of updates\n",
        "LR=1e-05                # Peak LR for polynomial LR scheduler.\n",
        "HEAD_NAME=task1a_head     # Custom name for the classification head.\n",
        "NUM_CLASSES=2           # Number of classes for the classification task.\n",
        "MAX_SENTENCES=8         # Batch size.\n",
        "ROBERTA_PATH=/content/fairseq/roberta.large/model.pt    #/content/fairseq/checkpoint/checkpoint_best.pt        \n",
        "CUDA_VISIBLE_DEVICES=0 fairseq-train /content/drive/MyDrive/UPENN/Task1a-bin/ \\\n",
        "    --restore-file $ROBERTA_PATH \\\n",
        "    --max-positions 512 \\\n",
        "    --batch-size $MAX_SENTENCES \\\n",
        "    --max-tokens 4400 \\\n",
        "    --task sentence_prediction \\\n",
        "    --reset-optimizer --reset-dataloader --reset-meters \\\n",
        "    --required-batch-size-multiple 1 \\\n",
        "    --init-token 0 --separator-token 2 \\\n",
        "    --arch roberta_large \\\n",
        "    --criterion sentence_prediction \\\n",
        "    --classification-head-name $HEAD_NAME \\\n",
        "    --num-classes $NUM_CLASSES \\\n",
        "    --dropout 0.1 --attention-dropout 0.1 \\\n",
        "    --weight-decay 0.1 --optimizer adam --adam-betas \"(0.9, 0.98)\" --adam-eps 1e-06 \\\n",
        "    --clip-norm 0.0 \\\n",
        "    --lr-scheduler polynomial_decay --lr $LR --total-num-update $TOTAL_NUM_UPDATES --warmup-updates $WARMUP_UPDATES \\\n",
        "    --fp16 --fp16-init-scale 4 --threshold-loss-scale 1 --fp16-scale-window 128 \\\n",
        "    --max-epoch 6 \\\n",
        "    --best-checkpoint-metric accuracy --maximize-best-checkpoint-metric \\\n",
        "    --shorten-method \"truncate\" \\\n",
        "    --find-unused-parameters \\\n",
        "    --update-freq 4"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-02-28 16:30:03 | INFO | fairseq_cli.train | Namespace(activation_dropout=0.0, activation_fn='gelu', adam_betas='(0.9, 0.98)', adam_eps=1e-06, add_prev_output_tokens=False, all_gather_list_size=16384, arch='roberta_large', attention_dropout=0.1, batch_size=8, batch_size_valid=8, best_checkpoint_metric='accuracy', bf16=False, bpe=None, broadcast_buffers=False, bucket_cap_mb=25, checkpoint_shard_count=1, checkpoint_suffix='', classification_head_name='task1a_head', clip_norm=0.0, cpu=False, criterion='sentence_prediction', curriculum=0, data='/content/drive/MyDrive/UPENN/Task1a-bin/', data_buffer_size=10, dataset_impl=None, ddp_backend='c10d', device_id=0, disable_validation=False, distributed_backend='nccl', distributed_init_method=None, distributed_no_spawn=False, distributed_num_procs=1, distributed_port=-1, distributed_rank=0, distributed_world_size=1, distributed_wrapper='DDP', dropout=0.1, empty_cache_freq=0, encoder_attention_heads=16, encoder_embed_dim=1024, encoder_ffn_embed_dim=4096, encoder_layerdrop=0, encoder_layers=24, encoder_layers_to_keep=None, end_learning_rate=0.0, fast_stat_sync=False, find_unused_parameters=True, finetune_from_model=None, fix_batches_to_gpus=False, fixed_validation_seed=None, force_anneal=None, fp16=True, fp16_init_scale=4, fp16_no_flatten_grads=False, fp16_scale_tolerance=0.0, fp16_scale_window=128, gen_subset='test', init_token=0, keep_best_checkpoints=-1, keep_interval_updates=-1, keep_last_epochs=-1, localsgd_frequency=3, log_format=None, log_interval=100, lr=[1e-05], lr_scheduler='polynomial_decay', max_epoch=6, max_positions=512, max_tokens=4400, max_tokens_valid=4400, max_update=0, maximize_best_checkpoint_metric=True, memory_efficient_bf16=False, memory_efficient_fp16=False, min_loss_scale=0.0001, min_lr=-1.0, model_parallel_size=1, no_epoch_checkpoints=False, no_last_checkpoints=False, no_progress_bar=False, no_save=False, no_save_optimizer_state=False, no_seed_provided=False, no_shuffle=False, nprocs_per_node=1, num_classes=2, num_shards=1, num_workers=1, optimizer='adam', optimizer_overrides='{}', patience=-1, pipeline_balance=None, pipeline_checkpoint='never', pipeline_chunks=0, pipeline_decoder_balance=None, pipeline_decoder_devices=None, pipeline_devices=None, pipeline_encoder_balance=None, pipeline_encoder_devices=None, pipeline_model_parallel=False, pooler_activation_fn='tanh', pooler_dropout=0.0, power=1.0, profile=False, quant_noise_pq=0, quant_noise_pq_block_size=8, quant_noise_scalar=0, quantization_config_path=None, regression_target=False, required_batch_size_multiple=1, required_seq_len_multiple=1, reset_dataloader=True, reset_lr_scheduler=False, reset_meters=True, reset_optimizer=True, restore_file='/content/fairseq/roberta.large/model.pt', save_dir='checkpoints', save_interval=1, save_interval_updates=0, scoring='bleu', seed=1, sentence_avg=False, separator_token=2, shard_id=0, shorten_data_split_list='', shorten_method='truncate', skip_invalid_size_inputs_valid_test=False, slowmo_algorithm='LocalSGD', slowmo_momentum=None, spectral_norm_classification_head=False, stop_time_hours=0, task='sentence_prediction', tensorboard_logdir=None, threshold_loss_scale=1.0, tokenizer=None, total_num_update=3614, tpu=False, train_subset='train', update_freq=[4], use_bmuf=False, use_old_adam=False, user_dir=None, valid_subset='valid', validate_after_updates=0, validate_interval=1, validate_interval_updates=0, warmup_updates=217, weight_decay=0.1, zero_sharding='none')\n",
            "2021-02-28 16:30:03 | INFO | fairseq.tasks.sentence_prediction | [input] dictionary: 50265 types\n",
            "2021-02-28 16:30:03 | INFO | fairseq.tasks.sentence_prediction | [label] dictionary: 9 types\n",
            "2021-02-28 16:30:03 | INFO | fairseq.data.data_utils | loaded 1826 examples from: /content/drive/MyDrive/UPENN/Task1a-bin/input0/valid\n",
            "2021-02-28 16:30:03 | INFO | fairseq.data.data_utils | loaded 1826 examples from: /content/drive/MyDrive/UPENN/Task1a-bin/label/valid\n",
            "2021-02-28 16:30:03 | INFO | fairseq.tasks.sentence_prediction | Loaded valid with #samples: 1826\n",
            "2021-02-28 16:30:18 | INFO | fairseq_cli.train | RobertaModel(\n",
            "  (encoder): RobertaEncoder(\n",
            "    (sentence_encoder): TransformerSentenceEncoder(\n",
            "      (dropout_module): FairseqDropout()\n",
            "      (embed_tokens): Embedding(50265, 1024, padding_idx=1)\n",
            "      (embed_positions): LearnedPositionalEmbedding(514, 1024, padding_idx=1)\n",
            "      (layers): ModuleList(\n",
            "        (0): TransformerSentenceEncoderLayer(\n",
            "          (dropout_module): FairseqDropout()\n",
            "          (activation_dropout_module): FairseqDropout()\n",
            "          (self_attn): MultiheadAttention(\n",
            "            (dropout_module): FairseqDropout()\n",
            "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "          )\n",
            "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
            "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
            "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "        )\n",
            "        (1): TransformerSentenceEncoderLayer(\n",
            "          (dropout_module): FairseqDropout()\n",
            "          (activation_dropout_module): FairseqDropout()\n",
            "          (self_attn): MultiheadAttention(\n",
            "            (dropout_module): FairseqDropout()\n",
            "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "          )\n",
            "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
            "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
            "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "        )\n",
            "        (2): TransformerSentenceEncoderLayer(\n",
            "          (dropout_module): FairseqDropout()\n",
            "          (activation_dropout_module): FairseqDropout()\n",
            "          (self_attn): MultiheadAttention(\n",
            "            (dropout_module): FairseqDropout()\n",
            "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "          )\n",
            "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
            "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
            "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "        )\n",
            "        (3): TransformerSentenceEncoderLayer(\n",
            "          (dropout_module): FairseqDropout()\n",
            "          (activation_dropout_module): FairseqDropout()\n",
            "          (self_attn): MultiheadAttention(\n",
            "            (dropout_module): FairseqDropout()\n",
            "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "          )\n",
            "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
            "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
            "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "        )\n",
            "        (4): TransformerSentenceEncoderLayer(\n",
            "          (dropout_module): FairseqDropout()\n",
            "          (activation_dropout_module): FairseqDropout()\n",
            "          (self_attn): MultiheadAttention(\n",
            "            (dropout_module): FairseqDropout()\n",
            "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "          )\n",
            "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
            "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
            "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "        )\n",
            "        (5): TransformerSentenceEncoderLayer(\n",
            "          (dropout_module): FairseqDropout()\n",
            "          (activation_dropout_module): FairseqDropout()\n",
            "          (self_attn): MultiheadAttention(\n",
            "            (dropout_module): FairseqDropout()\n",
            "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "          )\n",
            "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
            "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
            "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "        )\n",
            "        (6): TransformerSentenceEncoderLayer(\n",
            "          (dropout_module): FairseqDropout()\n",
            "          (activation_dropout_module): FairseqDropout()\n",
            "          (self_attn): MultiheadAttention(\n",
            "            (dropout_module): FairseqDropout()\n",
            "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "          )\n",
            "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
            "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
            "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "        )\n",
            "        (7): TransformerSentenceEncoderLayer(\n",
            "          (dropout_module): FairseqDropout()\n",
            "          (activation_dropout_module): FairseqDropout()\n",
            "          (self_attn): MultiheadAttention(\n",
            "            (dropout_module): FairseqDropout()\n",
            "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "          )\n",
            "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
            "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
            "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "        )\n",
            "        (8): TransformerSentenceEncoderLayer(\n",
            "          (dropout_module): FairseqDropout()\n",
            "          (activation_dropout_module): FairseqDropout()\n",
            "          (self_attn): MultiheadAttention(\n",
            "            (dropout_module): FairseqDropout()\n",
            "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "          )\n",
            "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
            "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
            "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "        )\n",
            "        (9): TransformerSentenceEncoderLayer(\n",
            "          (dropout_module): FairseqDropout()\n",
            "          (activation_dropout_module): FairseqDropout()\n",
            "          (self_attn): MultiheadAttention(\n",
            "            (dropout_module): FairseqDropout()\n",
            "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "          )\n",
            "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
            "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
            "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "        )\n",
            "        (10): TransformerSentenceEncoderLayer(\n",
            "          (dropout_module): FairseqDropout()\n",
            "          (activation_dropout_module): FairseqDropout()\n",
            "          (self_attn): MultiheadAttention(\n",
            "            (dropout_module): FairseqDropout()\n",
            "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "          )\n",
            "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
            "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
            "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "        )\n",
            "        (11): TransformerSentenceEncoderLayer(\n",
            "          (dropout_module): FairseqDropout()\n",
            "          (activation_dropout_module): FairseqDropout()\n",
            "          (self_attn): MultiheadAttention(\n",
            "            (dropout_module): FairseqDropout()\n",
            "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "          )\n",
            "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
            "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
            "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "        )\n",
            "        (12): TransformerSentenceEncoderLayer(\n",
            "          (dropout_module): FairseqDropout()\n",
            "          (activation_dropout_module): FairseqDropout()\n",
            "          (self_attn): MultiheadAttention(\n",
            "            (dropout_module): FairseqDropout()\n",
            "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "          )\n",
            "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
            "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
            "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "        )\n",
            "        (13): TransformerSentenceEncoderLayer(\n",
            "          (dropout_module): FairseqDropout()\n",
            "          (activation_dropout_module): FairseqDropout()\n",
            "          (self_attn): MultiheadAttention(\n",
            "            (dropout_module): FairseqDropout()\n",
            "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "          )\n",
            "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
            "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
            "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "        )\n",
            "        (14): TransformerSentenceEncoderLayer(\n",
            "          (dropout_module): FairseqDropout()\n",
            "          (activation_dropout_module): FairseqDropout()\n",
            "          (self_attn): MultiheadAttention(\n",
            "            (dropout_module): FairseqDropout()\n",
            "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "          )\n",
            "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
            "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
            "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "        )\n",
            "        (15): TransformerSentenceEncoderLayer(\n",
            "          (dropout_module): FairseqDropout()\n",
            "          (activation_dropout_module): FairseqDropout()\n",
            "          (self_attn): MultiheadAttention(\n",
            "            (dropout_module): FairseqDropout()\n",
            "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "          )\n",
            "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
            "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
            "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "        )\n",
            "        (16): TransformerSentenceEncoderLayer(\n",
            "          (dropout_module): FairseqDropout()\n",
            "          (activation_dropout_module): FairseqDropout()\n",
            "          (self_attn): MultiheadAttention(\n",
            "            (dropout_module): FairseqDropout()\n",
            "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "          )\n",
            "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
            "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
            "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "        )\n",
            "        (17): TransformerSentenceEncoderLayer(\n",
            "          (dropout_module): FairseqDropout()\n",
            "          (activation_dropout_module): FairseqDropout()\n",
            "          (self_attn): MultiheadAttention(\n",
            "            (dropout_module): FairseqDropout()\n",
            "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "          )\n",
            "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
            "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
            "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "        )\n",
            "        (18): TransformerSentenceEncoderLayer(\n",
            "          (dropout_module): FairseqDropout()\n",
            "          (activation_dropout_module): FairseqDropout()\n",
            "          (self_attn): MultiheadAttention(\n",
            "            (dropout_module): FairseqDropout()\n",
            "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "          )\n",
            "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
            "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
            "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "        )\n",
            "        (19): TransformerSentenceEncoderLayer(\n",
            "          (dropout_module): FairseqDropout()\n",
            "          (activation_dropout_module): FairseqDropout()\n",
            "          (self_attn): MultiheadAttention(\n",
            "            (dropout_module): FairseqDropout()\n",
            "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "          )\n",
            "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
            "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
            "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "        )\n",
            "        (20): TransformerSentenceEncoderLayer(\n",
            "          (dropout_module): FairseqDropout()\n",
            "          (activation_dropout_module): FairseqDropout()\n",
            "          (self_attn): MultiheadAttention(\n",
            "            (dropout_module): FairseqDropout()\n",
            "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "          )\n",
            "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
            "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
            "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "        )\n",
            "        (21): TransformerSentenceEncoderLayer(\n",
            "          (dropout_module): FairseqDropout()\n",
            "          (activation_dropout_module): FairseqDropout()\n",
            "          (self_attn): MultiheadAttention(\n",
            "            (dropout_module): FairseqDropout()\n",
            "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "          )\n",
            "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
            "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
            "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "        )\n",
            "        (22): TransformerSentenceEncoderLayer(\n",
            "          (dropout_module): FairseqDropout()\n",
            "          (activation_dropout_module): FairseqDropout()\n",
            "          (self_attn): MultiheadAttention(\n",
            "            (dropout_module): FairseqDropout()\n",
            "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "          )\n",
            "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
            "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
            "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "        )\n",
            "        (23): TransformerSentenceEncoderLayer(\n",
            "          (dropout_module): FairseqDropout()\n",
            "          (activation_dropout_module): FairseqDropout()\n",
            "          (self_attn): MultiheadAttention(\n",
            "            (dropout_module): FairseqDropout()\n",
            "            (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "            (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "          )\n",
            "          (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
            "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
            "          (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "        )\n",
            "      )\n",
            "      (emb_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "    )\n",
            "    (lm_head): RobertaLMHead(\n",
            "      (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "      (layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
            "    )\n",
            "  )\n",
            "  (classification_heads): ModuleDict(\n",
            "    (task1a_head): RobertaClassificationHead(\n",
            "      (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
            "      (dropout): Dropout(p=0.0, inplace=False)\n",
            "      (out_proj): Linear(in_features=1024, out_features=2, bias=True)\n",
            "    )\n",
            "  )\n",
            ")\n",
            "2021-02-28 16:30:18 | INFO | fairseq_cli.train | task: sentence_prediction (SentencePredictionTask)\n",
            "2021-02-28 16:30:18 | INFO | fairseq_cli.train | model: roberta_large (RobertaModel)\n",
            "2021-02-28 16:30:18 | INFO | fairseq_cli.train | criterion: sentence_prediction (SentencePredictionCriterion)\n",
            "2021-02-28 16:30:18 | INFO | fairseq_cli.train | num. model params: 356462683 (num. trained: 356462683)\n",
            "2021-02-28 16:30:27 | INFO | fairseq.trainer | detected shared parameter: encoder.sentence_encoder.embed_tokens.weight <- encoder.lm_head.weight\n",
            "2021-02-28 16:30:27 | INFO | fairseq.utils | ***********************CUDA enviroments for all 1 workers***********************\n",
            "2021-02-28 16:30:27 | INFO | fairseq.utils | rank   0: capabilities =  3.7  ; total memory = 11.173 GB ; name = Tesla K80                               \n",
            "2021-02-28 16:30:27 | INFO | fairseq.utils | ***********************CUDA enviroments for all 1 workers***********************\n",
            "2021-02-28 16:30:27 | INFO | fairseq_cli.train | training on 1 devices (GPUs/TPUs)\n",
            "2021-02-28 16:30:27 | INFO | fairseq_cli.train | max tokens per GPU = 4400 and max sentences per GPU = 8\n",
            "2021-02-28 16:30:33 | INFO | fairseq.models.roberta.model | Overwriting classification_heads.task1a_head.dense.weight\n",
            "2021-02-28 16:30:33 | INFO | fairseq.models.roberta.model | Overwriting classification_heads.task1a_head.dense.bias\n",
            "2021-02-28 16:30:33 | INFO | fairseq.models.roberta.model | Overwriting classification_heads.task1a_head.out_proj.weight\n",
            "2021-02-28 16:30:33 | INFO | fairseq.models.roberta.model | Overwriting classification_heads.task1a_head.out_proj.bias\n",
            "2021-02-28 16:30:33 | INFO | fairseq.trainer | loaded checkpoint /content/fairseq/roberta.large/model.pt (epoch 1 @ 0 updates)\n",
            "2021-02-28 16:30:33 | INFO | fairseq.trainer | NOTE: your device does NOT support faster training with --fp16, please switch to FP32 which is likely to be faster\n",
            "2021-02-28 16:30:33 | INFO | fairseq.trainer | loading train data for epoch 1\n",
            "2021-02-28 16:30:33 | INFO | fairseq.data.data_utils | loaded 4572 examples from: /content/drive/MyDrive/UPENN/Task1a-bin/input0/train\n",
            "2021-02-28 16:30:33 | INFO | fairseq.data.data_utils | loaded 4572 examples from: /content/drive/MyDrive/UPENN/Task1a-bin/label/train\n",
            "2021-02-28 16:30:33 | INFO | fairseq.tasks.sentence_prediction | Loaded train with #samples: 4572\n",
            "epoch 001:   0% 0/143 [00:00<?, ?it/s]2021-02-28 16:30:33 | INFO | fairseq.trainer | begin training epoch 1\n",
            "epoch 001:  99% 142/143 [09:30<00:04,  4.03s/it, loss=1.086, nll_loss=0.036, accuracy=45.5, wps=237.3, ups=0.25, wpb=954.3, bsz=32, num_updates=100, lr=4.60829e-06, gnorm=11.617, loss_scale=4, train_wall=402, wall=408]2021-02-28 16:40:08 | INFO | fairseq_cli.train | begin validation on \"valid\" subset\n",
            "\n",
            "epoch 001 | valid on 'valid' subset:   0% 0/229 [00:00<?, ?it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:   0% 1/229 [00:00<01:26,  2.62it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:   1% 2/229 [00:00<01:25,  2.67it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:   1% 3/229 [00:01<01:19,  2.83it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:   2% 4/229 [00:01<01:17,  2.89it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:   2% 5/229 [00:01<01:16,  2.94it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:   3% 6/229 [00:02<01:15,  2.96it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:   3% 7/229 [00:02<01:14,  2.99it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:   3% 8/229 [00:02<01:11,  3.08it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:   4% 9/229 [00:02<01:12,  3.05it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:   4% 10/229 [00:03<01:10,  3.13it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:   5% 11/229 [00:03<01:12,  3.02it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:   5% 12/229 [00:03<01:12,  3.01it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:   6% 13/229 [00:04<01:09,  3.09it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:   6% 14/229 [00:04<01:10,  3.07it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:   7% 15/229 [00:04<01:10,  3.05it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:   7% 16/229 [00:05<01:10,  3.04it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:   7% 17/229 [00:05<01:08,  3.11it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:   8% 18/229 [00:05<01:10,  3.00it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:   8% 19/229 [00:06<01:11,  2.93it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:   9% 20/229 [00:06<01:08,  3.03it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:   9% 21/229 [00:06<01:08,  3.03it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  10% 22/229 [00:07<01:06,  3.10it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  10% 23/229 [00:07<01:05,  3.16it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  10% 24/229 [00:07<01:05,  3.11it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  11% 25/229 [00:08<01:06,  3.09it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  11% 26/229 [00:08<01:06,  3.07it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  12% 27/229 [00:08<01:04,  3.13it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  12% 28/229 [00:09<01:04,  3.11it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  13% 29/229 [00:09<01:04,  3.08it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  13% 30/229 [00:09<01:03,  3.15it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  14% 31/229 [00:10<01:03,  3.11it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  14% 32/229 [00:10<01:04,  3.08it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  14% 33/229 [00:10<01:02,  3.14it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  15% 34/229 [00:11<01:04,  3.02it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  15% 35/229 [00:11<01:06,  2.94it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  16% 36/229 [00:11<01:03,  3.04it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  16% 37/229 [00:12<01:03,  3.03it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  17% 38/229 [00:12<01:03,  3.03it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  17% 39/229 [00:12<01:02,  3.02it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  17% 40/229 [00:13<01:00,  3.10it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  18% 41/229 [00:13<01:01,  3.07it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  18% 42/229 [00:13<01:02,  2.97it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  19% 43/229 [00:14<01:05,  2.84it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  19% 44/229 [00:14<01:04,  2.88it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  20% 45/229 [00:14<01:01,  3.00it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  20% 46/229 [00:15<01:02,  2.92it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  21% 47/229 [00:15<01:03,  2.88it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  21% 48/229 [00:15<01:05,  2.76it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  21% 49/229 [00:16<01:06,  2.69it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  22% 50/229 [00:16<01:04,  2.78it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  22% 51/229 [00:17<01:03,  2.78it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  23% 52/229 [00:17<01:02,  2.84it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  23% 53/229 [00:17<01:00,  2.90it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  24% 54/229 [00:18<00:59,  2.93it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  24% 55/229 [00:18<00:57,  3.03it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  24% 56/229 [00:18<00:57,  3.03it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  25% 57/229 [00:18<00:56,  3.03it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  25% 58/229 [00:19<00:55,  3.10it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  26% 59/229 [00:19<00:55,  3.08it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  26% 60/229 [00:19<00:55,  3.06it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  27% 61/229 [00:20<00:51,  3.26it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  27% 62/229 [00:20<00:52,  3.19it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  28% 63/229 [00:20<00:53,  3.13it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  28% 64/229 [00:21<00:51,  3.18it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  28% 65/229 [00:21<00:52,  3.13it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  29% 66/229 [00:21<00:52,  3.10it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  29% 67/229 [00:22<00:54,  3.00it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  30% 68/229 [00:22<00:53,  2.99it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  30% 69/229 [00:22<00:53,  3.00it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  31% 70/229 [00:23<00:53,  2.98it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  31% 71/229 [00:23<00:52,  2.99it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  31% 72/229 [00:23<00:52,  3.01it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  32% 73/229 [00:24<00:51,  3.02it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  32% 74/229 [00:24<00:51,  3.02it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  33% 75/229 [00:24<00:51,  3.02it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  33% 76/229 [00:25<00:50,  3.02it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  34% 77/229 [00:25<00:50,  3.02it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  34% 78/229 [00:25<00:49,  3.02it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  34% 79/229 [00:26<00:48,  3.09it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  35% 80/229 [00:26<00:49,  3.00it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  35% 81/229 [00:26<00:48,  3.08it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  36% 82/229 [00:27<00:48,  3.05it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  36% 83/229 [00:27<00:46,  3.12it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  37% 84/229 [00:27<00:45,  3.17it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  37% 85/229 [00:28<00:45,  3.20it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  38% 86/229 [00:28<00:45,  3.14it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  38% 87/229 [00:28<00:45,  3.11it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  38% 88/229 [00:29<00:46,  3.00it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  39% 89/229 [00:29<00:45,  3.09it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  39% 90/229 [00:29<00:46,  2.99it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  40% 91/229 [00:30<00:46,  3.00it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  40% 92/229 [00:30<00:44,  3.08it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  41% 93/229 [00:30<00:44,  3.05it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  41% 94/229 [00:31<00:44,  3.04it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  41% 95/229 [00:31<00:44,  3.03it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  42% 96/229 [00:31<00:43,  3.03it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  42% 97/229 [00:32<00:43,  3.02it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  43% 98/229 [00:32<00:43,  3.03it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  43% 99/229 [00:32<00:42,  3.03it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  44% 100/229 [00:33<00:41,  3.10it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  44% 101/229 [00:33<00:40,  3.16it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  45% 102/229 [00:33<00:40,  3.12it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  45% 103/229 [00:34<00:41,  3.01it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  45% 104/229 [00:34<00:42,  2.94it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  46% 105/229 [00:34<00:41,  2.96it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  46% 106/229 [00:35<00:41,  2.97it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  47% 107/229 [00:35<00:40,  2.98it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  47% 108/229 [00:35<00:40,  2.99it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  48% 109/229 [00:35<00:39,  3.08it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  48% 110/229 [00:36<00:38,  3.07it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  48% 111/229 [00:36<00:39,  2.97it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  49% 112/229 [00:36<00:38,  3.06it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  49% 113/229 [00:37<00:37,  3.13it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  50% 114/229 [00:37<00:37,  3.09it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  50% 115/229 [00:37<00:38,  2.99it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  51% 116/229 [00:38<00:37,  2.99it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  51% 117/229 [00:38<00:38,  2.92it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  52% 118/229 [00:39<00:38,  2.87it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  52% 119/229 [00:39<00:36,  2.99it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  52% 120/229 [00:39<00:36,  3.01it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  53% 121/229 [00:40<00:35,  3.02it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  53% 122/229 [00:40<00:35,  3.02it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  54% 123/229 [00:40<00:35,  3.02it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  54% 124/229 [00:40<00:33,  3.10it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  55% 125/229 [00:41<00:35,  2.91it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  55% 126/229 [00:41<00:35,  2.87it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  55% 127/229 [00:42<00:35,  2.91it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  56% 128/229 [00:42<00:35,  2.87it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  56% 129/229 [00:42<00:34,  2.92it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  57% 130/229 [00:43<00:33,  2.94it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  57% 131/229 [00:43<00:34,  2.82it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  58% 132/229 [00:43<00:33,  2.86it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  58% 133/229 [00:44<00:33,  2.91it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  59% 134/229 [00:44<00:32,  2.94it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  59% 135/229 [00:44<00:31,  2.96it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  59% 136/229 [00:45<00:30,  3.04it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  60% 137/229 [00:45<00:31,  2.96it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  60% 138/229 [00:45<00:29,  3.05it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  61% 139/229 [00:46<00:29,  3.03it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  61% 140/229 [00:46<00:28,  3.10it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  62% 141/229 [00:46<00:27,  3.16it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  62% 142/229 [00:47<00:27,  3.12it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  62% 143/229 [00:47<00:28,  3.00it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  63% 144/229 [00:47<00:29,  2.93it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  63% 145/229 [00:48<00:30,  2.71it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  64% 146/229 [00:48<00:30,  2.73it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  64% 147/229 [00:48<00:29,  2.81it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  65% 148/229 [00:49<00:28,  2.86it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  65% 149/229 [00:49<00:28,  2.84it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  66% 150/229 [00:49<00:26,  2.96it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  66% 151/229 [00:50<00:26,  2.97it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  66% 152/229 [00:50<00:25,  2.98it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  67% 153/229 [00:50<00:24,  3.08it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  67% 154/229 [00:51<00:24,  3.05it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  68% 155/229 [00:51<00:24,  3.04it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  68% 156/229 [00:51<00:24,  3.03it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  69% 157/229 [00:52<00:23,  3.02it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  69% 158/229 [00:52<00:23,  3.01it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  69% 159/229 [00:52<00:22,  3.10it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  70% 160/229 [00:53<00:21,  3.14it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  70% 161/229 [00:53<00:21,  3.10it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  71% 162/229 [00:53<00:21,  3.16it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  71% 163/229 [00:54<00:21,  3.11it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  72% 164/229 [00:54<00:21,  2.99it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  72% 165/229 [00:54<00:20,  3.07it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  72% 166/229 [00:55<00:20,  3.05it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  73% 167/229 [00:55<00:19,  3.12it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  73% 168/229 [00:55<00:19,  3.08it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  74% 169/229 [00:56<00:19,  3.14it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  74% 170/229 [00:56<00:19,  3.02it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  75% 171/229 [00:56<00:18,  3.09it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  75% 172/229 [00:57<00:18,  3.07it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  76% 173/229 [00:57<00:17,  3.14it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  76% 174/229 [00:57<00:17,  3.10it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  76% 175/229 [00:57<00:17,  3.15it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  77% 176/229 [00:58<00:15,  3.32it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  77% 177/229 [00:58<00:16,  3.22it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  78% 178/229 [00:58<00:16,  3.07it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  78% 179/229 [00:59<00:15,  3.14it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  79% 180/229 [00:59<00:15,  3.18it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  79% 181/229 [00:59<00:15,  3.13it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  79% 182/229 [01:00<00:14,  3.17it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  80% 183/229 [01:00<00:14,  3.21it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  80% 184/229 [01:00<00:13,  3.24it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  81% 185/229 [01:01<00:13,  3.17it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  81% 186/229 [01:01<00:13,  3.23it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  82% 187/229 [01:01<00:13,  3.17it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  82% 188/229 [01:02<00:13,  3.04it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  83% 189/229 [01:02<00:13,  2.95it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  83% 190/229 [01:02<00:13,  2.97it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  83% 191/229 [01:03<00:12,  2.99it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  84% 192/229 [01:03<00:11,  3.10it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  84% 193/229 [01:03<00:11,  3.15it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  85% 194/229 [01:04<00:10,  3.19it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  85% 195/229 [01:04<00:10,  3.14it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  86% 196/229 [01:04<00:10,  3.18it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  86% 197/229 [01:05<00:10,  3.04it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  86% 198/229 [01:05<00:09,  3.11it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  87% 199/229 [01:05<00:09,  3.16it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  87% 200/229 [01:05<00:09,  3.12it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  88% 201/229 [01:06<00:09,  3.08it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  88% 202/229 [01:06<00:09,  2.89it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  89% 203/229 [01:06<00:08,  3.00it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  89% 204/229 [01:07<00:08,  3.01it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  90% 205/229 [01:07<00:07,  3.08it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  90% 206/229 [01:08<00:07,  2.97it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  90% 207/229 [01:08<00:07,  2.98it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  91% 208/229 [01:08<00:06,  3.07it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  91% 209/229 [01:08<00:06,  3.14it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  92% 210/229 [01:09<00:06,  3.07it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  92% 211/229 [01:09<00:05,  3.05it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  93% 212/229 [01:09<00:05,  3.12it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  93% 213/229 [01:10<00:05,  3.01it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  93% 214/229 [01:10<00:04,  3.01it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  94% 215/229 [01:10<00:04,  2.93it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  94% 216/229 [01:11<00:04,  3.02it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  95% 217/229 [01:11<00:04,  2.94it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  95% 218/229 [01:11<00:03,  2.96it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  96% 219/229 [01:12<00:03,  2.98it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  96% 220/229 [01:12<00:02,  3.01it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  97% 221/229 [01:12<00:02,  3.01it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  97% 222/229 [01:13<00:02,  2.95it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  97% 223/229 [01:13<00:01,  3.05it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  98% 224/229 [01:13<00:01,  2.97it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  98% 225/229 [01:14<00:01,  2.91it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  99% 226/229 [01:14<00:00,  3.04it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset:  99% 227/229 [01:14<00:00,  3.12it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset: 100% 228/229 [01:15<00:00,  3.19it/s]\u001b[A\n",
            "epoch 001 | valid on 'valid' subset: 100% 229/229 [01:15<00:00,  3.97it/s]\u001b[A\n",
            "                                                                          \u001b[A2021-02-28 16:41:23 | INFO | valid | epoch 001 | valid on 'valid' subset | loss 0.381 | nll_loss 0.013 | accuracy 90.6 | wps 706.7 | wpb 232.7 | bsz 8 | num_updates 143\n",
            "2021-02-28 16:41:23 | INFO | fairseq_cli.train | begin save checkpoint\n",
            "tcmalloc: large alloc 1425858560 bytes == 0x5649e8214000 @  0x7f59d375cb6b 0x7f59d377c379 0x7f597693574e 0x7f59769377b6 0x7f59b13a1e43 0x7f59b0d8c7ff 0x7f59b10a3bdc 0x7f59b104f24b 0x7f59b106e065 0x7f59b1049a7b 0x7f59b104f24b 0x7f59b106e065 0x7f59b11381ee 0x7f59b264ad9e 0x7f59b104f24b 0x7f59b106e065 0x7f59b1049a7b 0x7f59b104f24b 0x7f59b106e065 0x7f59b11381ee 0x7f59b0d7a840 0x7f59b12f3323 0x7f59b08f7a68 0x7f59b1075573 0x7f59b13740e9 0x7f59c0ce0b02 0x7f59c0df9a9c 0x56494128d050 0x56494137e99d 0x564941300fe9 0x5649412fbb0e\n",
            "tcmalloc: large alloc 1425858560 bytes == 0x564a3d1e2000 @  0x7f59d375cb6b 0x7f59d377c379 0x7f597693574e 0x7f59769377b6 0x7f59b13a1e43 0x7f59b0d8c7ff 0x7f59b10a3bdc 0x7f59b104f24b 0x7f59b106e065 0x7f59b1049a7b 0x7f59b104f24b 0x7f59b106e065 0x7f59b11381ee 0x7f59b264ad9e 0x7f59b104f24b 0x7f59b106e065 0x7f59b1049a7b 0x7f59b104f24b 0x7f59b106e065 0x7f59b11381ee 0x7f59b0d7a840 0x7f59b12f3323 0x7f59b08f7a68 0x7f59b1075573 0x7f59b13740e9 0x7f59c0ce0b02 0x7f59c0df9a9c 0x56494128d050 0x56494137e99d 0x564941300fe9 0x5649412fbb0e\n",
            "tcmalloc: large alloc 1425858560 bytes == 0x564a9fe0e000 @  0x7f59d377a1e7 0x5649412bef48 0x5649412899c7 0x7f59c118a3f7 0x7f59b1ad0465 0x7f59b1acc9ca 0x7f59b1ad1609 0x7f59c118e00e 0x7f59c0e130a0 0x56494128dc38 0x56494130163d 0x5649412fbe0d 0x56494128e77a 0x5649412fca45 0x5649412fbb0e 0x56494128e77a 0x564941300e50 0x56494128e69a 0x5649412fca45 0x5649412fbb0e 0x56494128e77a 0x564941300e50 0x56494128e69a 0x5649412fcc9e 0x5649412fbe0d 0x56494128e77a 0x564941300e50 0x56494128e69a 0x5649412fca45 0x5649411cdd14 0x5649412fe1e6\n",
            "2021-02-28 16:47:04 | INFO | fairseq.checkpoint_utils | saved checkpoint checkpoints/checkpoint1.pt (epoch 1 @ 143 updates, score 90.6) (writing took 341.21901983700013 seconds)\n",
            "2021-02-28 16:47:05 | INFO | fairseq_cli.train | end of epoch 1 (average epoch stats below)\n",
            "2021-02-28 16:47:05 | INFO | train | epoch 001 | loss 0.989 | nll_loss 0.033 | accuracy 53.4 | wps 137.7 | ups 0.14 | wpb 957.5 | bsz 32 | num_updates 143 | lr 6.58986e-06 | gnorm 11.184 | loss_scale 8 | train_wall 574 | wall 998\n",
            "epoch 002:   0% 0/143 [00:00<?, ?it/s]2021-02-28 16:47:05 | INFO | fairseq.trainer | begin training epoch 2\n",
            "epoch 002:  99% 142/143 [09:30<00:04,  4.04s/it, loss=0.614, nll_loss=0.02, accuracy=79.7, wps=117.7, ups=0.12, wpb=965.2, bsz=31.9, num_updates=200, lr=9.21659e-06, gnorm=14.105, loss_scale=8, train_wall=403, wall=1228]2021-02-28 16:56:39 | INFO | fairseq_cli.train | begin validation on \"valid\" subset\n",
            "\n",
            "epoch 002 | valid on 'valid' subset:   0% 0/229 [00:00<?, ?it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:   0% 1/229 [00:00<01:28,  2.57it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:   1% 2/229 [00:00<01:26,  2.63it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:   1% 3/229 [00:01<01:20,  2.80it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:   2% 4/229 [00:01<01:18,  2.86it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:   2% 5/229 [00:01<01:16,  2.91it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:   3% 6/229 [00:02<01:15,  2.94it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:   3% 7/229 [00:02<01:14,  2.97it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:   3% 8/229 [00:02<01:12,  3.07it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:   4% 9/229 [00:03<01:12,  3.05it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:   4% 10/229 [00:03<01:09,  3.13it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:   5% 11/229 [00:03<01:12,  3.02it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:   5% 12/229 [00:03<01:11,  3.01it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:   6% 13/229 [00:04<01:09,  3.10it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:   6% 14/229 [00:04<01:09,  3.07it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:   7% 15/229 [00:04<01:10,  3.06it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:   7% 16/229 [00:05<01:09,  3.05it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:   7% 17/229 [00:05<01:08,  3.11it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:   8% 18/229 [00:05<01:10,  3.00it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:   8% 19/229 [00:06<01:11,  2.92it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:   9% 20/229 [00:06<01:08,  3.03it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:   9% 21/229 [00:06<01:08,  3.02it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  10% 22/229 [00:07<01:06,  3.09it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  10% 23/229 [00:07<01:05,  3.15it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  10% 24/229 [00:07<01:06,  3.10it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  11% 25/229 [00:08<01:06,  3.08it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  11% 26/229 [00:08<01:06,  3.05it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  12% 27/229 [00:08<01:04,  3.12it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  12% 28/229 [00:09<01:04,  3.10it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  13% 29/229 [00:09<01:05,  3.07it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  13% 30/229 [00:09<01:03,  3.14it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  14% 31/229 [00:10<01:03,  3.10it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  14% 32/229 [00:10<01:04,  3.07it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  14% 33/229 [00:10<01:02,  3.13it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  15% 34/229 [00:11<01:04,  3.01it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  15% 35/229 [00:11<01:06,  2.93it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  16% 36/229 [00:11<01:03,  3.04it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  16% 37/229 [00:12<01:03,  3.03it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  17% 38/229 [00:12<01:03,  3.03it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  17% 39/229 [00:12<01:02,  3.02it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  17% 40/229 [00:13<01:00,  3.10it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  18% 41/229 [00:13<01:01,  3.06it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  18% 42/229 [00:13<01:03,  2.96it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  19% 43/229 [00:14<01:05,  2.83it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  19% 44/229 [00:14<01:04,  2.87it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  20% 45/229 [00:14<01:01,  2.99it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  20% 46/229 [00:15<01:02,  2.92it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  21% 47/229 [00:15<01:03,  2.87it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  21% 48/229 [00:15<01:05,  2.76it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  21% 49/229 [00:16<01:06,  2.69it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  22% 50/229 [00:16<01:04,  2.77it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  22% 51/229 [00:17<01:04,  2.78it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  23% 52/229 [00:17<01:02,  2.84it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  23% 53/229 [00:17<01:00,  2.89it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  24% 54/229 [00:18<00:59,  2.92it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  24% 55/229 [00:18<00:57,  3.02it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  24% 56/229 [00:18<00:57,  3.02it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  25% 57/229 [00:19<00:56,  3.02it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  25% 58/229 [00:19<00:55,  3.09it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  26% 59/229 [00:19<00:55,  3.06it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  26% 60/229 [00:20<00:55,  3.04it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  27% 61/229 [00:20<00:51,  3.24it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  27% 62/229 [00:20<00:52,  3.17it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  28% 63/229 [00:20<00:53,  3.11it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  28% 64/229 [00:21<00:52,  3.16it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  28% 65/229 [00:21<00:52,  3.11it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  29% 66/229 [00:21<00:52,  3.08it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  29% 67/229 [00:22<00:54,  2.98it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  30% 68/229 [00:22<00:53,  2.98it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  30% 69/229 [00:22<00:53,  2.99it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  31% 70/229 [00:23<00:53,  2.99it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  31% 71/229 [00:23<00:52,  3.00it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  31% 72/229 [00:23<00:52,  3.00it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  32% 73/229 [00:24<00:51,  3.01it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  32% 74/229 [00:24<00:51,  3.01it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  33% 75/229 [00:24<00:51,  3.01it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  33% 76/229 [00:25<00:50,  3.01it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  34% 77/229 [00:25<00:50,  3.01it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  34% 78/229 [00:25<00:50,  3.01it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  34% 79/229 [00:26<00:48,  3.09it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  35% 80/229 [00:26<00:49,  2.99it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  35% 81/229 [00:26<00:48,  3.07it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  36% 82/229 [00:27<00:48,  3.05it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  36% 83/229 [00:27<00:46,  3.11it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  37% 84/229 [00:27<00:45,  3.17it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  37% 85/229 [00:28<00:44,  3.20it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  38% 86/229 [00:28<00:45,  3.15it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  38% 87/229 [00:28<00:45,  3.11it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  38% 88/229 [00:29<00:46,  3.01it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  39% 89/229 [00:29<00:45,  3.10it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  39% 90/229 [00:29<00:46,  2.99it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  40% 91/229 [00:30<00:45,  3.01it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  40% 92/229 [00:30<00:44,  3.08it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  41% 93/229 [00:30<00:44,  3.06it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  41% 94/229 [00:31<00:44,  3.04it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  41% 95/229 [00:31<00:44,  3.03it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  42% 96/229 [00:31<00:43,  3.03it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  42% 97/229 [00:32<00:43,  3.01it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  43% 98/229 [00:32<00:43,  3.02it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  43% 99/229 [00:32<00:42,  3.03it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  44% 100/229 [00:33<00:41,  3.10it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  44% 101/229 [00:33<00:40,  3.15it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  45% 102/229 [00:33<00:40,  3.11it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  45% 103/229 [00:34<00:42,  2.99it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  45% 104/229 [00:34<00:42,  2.93it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  46% 105/229 [00:34<00:42,  2.95it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  46% 106/229 [00:35<00:41,  2.96it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  47% 107/229 [00:35<00:41,  2.97it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  47% 108/229 [00:35<00:40,  2.98it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  48% 109/229 [00:36<00:39,  3.06it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  48% 110/229 [00:36<00:38,  3.05it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  48% 111/229 [00:36<00:39,  2.96it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  49% 112/229 [00:37<00:38,  3.06it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  49% 113/229 [00:37<00:37,  3.13it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  50% 114/229 [00:37<00:37,  3.09it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  50% 115/229 [00:38<00:38,  2.98it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  51% 116/229 [00:38<00:37,  2.99it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  51% 117/229 [00:38<00:38,  2.91it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  52% 118/229 [00:39<00:38,  2.86it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  52% 119/229 [00:39<00:36,  2.98it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  52% 120/229 [00:39<00:36,  3.00it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  53% 121/229 [00:40<00:35,  3.01it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  53% 122/229 [00:40<00:35,  3.01it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  54% 123/229 [00:40<00:35,  3.02it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  54% 124/229 [00:41<00:33,  3.09it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  55% 125/229 [00:41<00:35,  2.89it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  55% 126/229 [00:41<00:36,  2.86it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  55% 127/229 [00:42<00:35,  2.90it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  56% 128/229 [00:42<00:35,  2.85it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  56% 129/229 [00:42<00:34,  2.91it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  57% 130/229 [00:43<00:33,  2.93it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  57% 131/229 [00:43<00:34,  2.80it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  58% 132/229 [00:43<00:33,  2.86it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  58% 133/229 [00:44<00:33,  2.90it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  59% 134/229 [00:44<00:32,  2.94it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  59% 135/229 [00:44<00:31,  2.96it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  59% 136/229 [00:45<00:30,  3.05it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  60% 137/229 [00:45<00:31,  2.96it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  60% 138/229 [00:45<00:29,  3.05it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  61% 139/229 [00:46<00:29,  3.03it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  61% 140/229 [00:46<00:28,  3.11it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  62% 141/229 [00:46<00:27,  3.16it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  62% 142/229 [00:47<00:27,  3.12it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  62% 143/229 [00:47<00:28,  3.00it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  63% 144/229 [00:47<00:29,  2.93it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  63% 145/229 [00:48<00:30,  2.71it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  64% 146/229 [00:48<00:30,  2.73it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  64% 147/229 [00:48<00:29,  2.81it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  65% 148/229 [00:49<00:28,  2.87it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  65% 149/229 [00:49<00:28,  2.84it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  66% 150/229 [00:49<00:26,  2.96it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  66% 151/229 [00:50<00:26,  2.98it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  66% 152/229 [00:50<00:25,  2.99it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  67% 153/229 [00:50<00:24,  3.08it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  67% 154/229 [00:51<00:24,  3.06it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  68% 155/229 [00:51<00:24,  3.05it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  68% 156/229 [00:51<00:23,  3.04it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  69% 157/229 [00:52<00:23,  3.04it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  69% 158/229 [00:52<00:23,  3.03it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  69% 159/229 [00:52<00:22,  3.11it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  70% 160/229 [00:53<00:21,  3.16it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  70% 161/229 [00:53<00:21,  3.12it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  71% 162/229 [00:53<00:21,  3.18it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  71% 163/229 [00:54<00:21,  3.13it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  72% 164/229 [00:54<00:21,  3.01it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  72% 165/229 [00:54<00:20,  3.09it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  72% 166/229 [00:55<00:20,  3.08it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  73% 167/229 [00:55<00:19,  3.13it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  73% 168/229 [00:55<00:19,  3.10it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  74% 169/229 [00:56<00:19,  3.16it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  74% 170/229 [00:56<00:19,  3.03it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  75% 171/229 [00:56<00:18,  3.10it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  75% 172/229 [00:57<00:18,  3.08it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  76% 173/229 [00:57<00:17,  3.14it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  76% 174/229 [00:57<00:17,  3.10it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  76% 175/229 [00:58<00:17,  3.16it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  77% 176/229 [00:58<00:15,  3.34it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  77% 177/229 [00:58<00:16,  3.24it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  78% 178/229 [00:58<00:16,  3.09it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  78% 179/229 [00:59<00:15,  3.15it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  79% 180/229 [00:59<00:15,  3.19it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  79% 181/229 [00:59<00:15,  3.13it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  79% 182/229 [01:00<00:14,  3.18it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  80% 183/229 [01:00<00:14,  3.22it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  80% 184/229 [01:00<00:13,  3.25it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  81% 185/229 [01:01<00:13,  3.18it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  81% 186/229 [01:01<00:13,  3.23it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  82% 187/229 [01:01<00:13,  3.17it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  82% 188/229 [01:02<00:13,  3.04it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  83% 189/229 [01:02<00:13,  2.95it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  83% 190/229 [01:02<00:13,  2.97it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  83% 191/229 [01:03<00:12,  2.99it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  84% 192/229 [01:03<00:11,  3.09it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  84% 193/229 [01:03<00:11,  3.14it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  85% 194/229 [01:04<00:10,  3.19it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  85% 195/229 [01:04<00:10,  3.15it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  86% 196/229 [01:04<00:10,  3.19it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  86% 197/229 [01:05<00:10,  3.05it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  86% 198/229 [01:05<00:09,  3.11it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  87% 199/229 [01:05<00:09,  3.16it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  87% 200/229 [01:06<00:09,  3.12it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  88% 201/229 [01:06<00:09,  3.08it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  88% 202/229 [01:06<00:09,  2.89it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  89% 203/229 [01:07<00:08,  3.01it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  89% 204/229 [01:07<00:08,  3.01it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  90% 205/229 [01:07<00:07,  3.09it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  90% 206/229 [01:08<00:07,  2.98it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  90% 207/229 [01:08<00:07,  2.99it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  91% 208/229 [01:08<00:06,  3.06it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  91% 209/229 [01:08<00:06,  3.13it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  92% 210/229 [01:09<00:06,  3.09it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  92% 211/229 [01:09<00:05,  3.06it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  93% 212/229 [01:09<00:05,  3.13it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  93% 213/229 [01:10<00:05,  3.01it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  93% 214/229 [01:10<00:04,  3.01it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  94% 215/229 [01:10<00:04,  2.94it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  94% 216/229 [01:11<00:04,  3.03it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  95% 217/229 [01:11<00:04,  2.95it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  95% 218/229 [01:11<00:03,  2.96it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  96% 219/229 [01:12<00:03,  2.98it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  96% 220/229 [01:12<00:02,  3.00it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  97% 221/229 [01:12<00:02,  3.01it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  97% 222/229 [01:13<00:02,  2.95it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  97% 223/229 [01:13<00:01,  3.05it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  98% 224/229 [01:14<00:01,  2.96it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  98% 225/229 [01:14<00:01,  2.91it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  99% 226/229 [01:14<00:00,  3.04it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset:  99% 227/229 [01:14<00:00,  3.12it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset: 100% 228/229 [01:15<00:00,  3.19it/s]\u001b[A\n",
            "epoch 002 | valid on 'valid' subset: 100% 229/229 [01:15<00:00,  3.98it/s]\u001b[A\n",
            "                                                                          \u001b[A2021-02-28 16:57:55 | INFO | valid | epoch 002 | valid on 'valid' subset | loss 0.343 | nll_loss 0.012 | accuracy 89.8 | wps 706.6 | wpb 232.7 | bsz 8 | num_updates 286 | best_accuracy 90.6\n",
            "2021-02-28 16:57:55 | INFO | fairseq_cli.train | begin save checkpoint\n",
            "2021-02-28 17:01:28 | INFO | fairseq.checkpoint_utils | saved checkpoint checkpoints/checkpoint2.pt (epoch 2 @ 286 updates, score 89.8) (writing took 213.32354600899998 seconds)\n",
            "2021-02-28 17:01:28 | INFO | fairseq_cli.train | end of epoch 2 (average epoch stats below)\n",
            "2021-02-28 17:01:28 | INFO | train | epoch 002 | loss 0.399 | nll_loss 0.013 | accuracy 88.8 | wps 158.6 | ups 0.17 | wpb 957.5 | bsz 32 | num_updates 286 | lr 9.79688e-06 | gnorm 17.138 | loss_scale 16 | train_wall 574 | wall 1861\n",
            "epoch 003:   0% 0/143 [00:00<?, ?it/s]2021-02-28 17:01:28 | INFO | fairseq.trainer | begin training epoch 3\n",
            "epoch 003:  99% 142/143 [09:30<00:04,  4.04s/it, loss=0.203, nll_loss=0.007, accuracy=95.3, wps=237.8, ups=0.25, wpb=957.3, bsz=32, num_updates=400, lr=9.46129e-06, gnorm=15.506, loss_scale=32, train_wall=402, wall=2320]2021-02-28 17:11:03 | INFO | fairseq_cli.train | begin validation on \"valid\" subset\n",
            "\n",
            "epoch 003 | valid on 'valid' subset:   0% 0/229 [00:00<?, ?it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:   0% 1/229 [00:00<01:26,  2.63it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:   1% 2/229 [00:00<01:24,  2.68it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:   1% 3/229 [00:01<01:19,  2.84it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:   2% 4/229 [00:01<01:17,  2.90it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:   2% 5/229 [00:01<01:16,  2.94it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:   3% 6/229 [00:02<01:15,  2.96it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:   3% 7/229 [00:02<01:14,  2.98it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:   3% 8/229 [00:02<01:11,  3.08it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:   4% 9/229 [00:02<01:11,  3.06it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:   4% 10/229 [00:03<01:09,  3.13it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:   5% 11/229 [00:03<01:12,  3.02it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:   5% 12/229 [00:03<01:11,  3.02it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:   6% 13/229 [00:04<01:09,  3.10it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:   6% 14/229 [00:04<01:10,  3.06it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:   7% 15/229 [00:04<01:10,  3.04it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:   7% 16/229 [00:05<01:10,  3.02it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:   7% 17/229 [00:05<01:08,  3.10it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:   8% 18/229 [00:05<01:10,  2.99it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:   8% 19/229 [00:06<01:11,  2.92it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:   9% 20/229 [00:06<01:08,  3.03it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:   9% 21/229 [00:06<01:08,  3.02it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  10% 22/229 [00:07<01:06,  3.10it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  10% 23/229 [00:07<01:05,  3.16it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  10% 24/229 [00:07<01:05,  3.12it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  11% 25/229 [00:08<01:06,  3.09it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  11% 26/229 [00:08<01:06,  3.06it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  12% 27/229 [00:08<01:04,  3.13it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  12% 28/229 [00:09<01:04,  3.10it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  13% 29/229 [00:09<01:05,  3.07it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  13% 30/229 [00:09<01:03,  3.14it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  14% 31/229 [00:10<01:03,  3.10it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  14% 32/229 [00:10<01:04,  3.08it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  14% 33/229 [00:10<01:02,  3.14it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  15% 34/229 [00:11<01:04,  3.02it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  15% 35/229 [00:11<01:05,  2.94it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  16% 36/229 [00:11<01:03,  3.05it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  16% 37/229 [00:12<01:03,  3.04it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  17% 38/229 [00:12<01:02,  3.04it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  17% 39/229 [00:12<01:02,  3.03it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  17% 40/229 [00:13<01:00,  3.11it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  18% 41/229 [00:13<01:01,  3.08it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  18% 42/229 [00:13<01:02,  2.98it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  19% 43/229 [00:14<01:05,  2.84it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  19% 44/229 [00:14<01:04,  2.88it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  20% 45/229 [00:14<01:01,  2.99it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  20% 46/229 [00:15<01:02,  2.91it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  21% 47/229 [00:15<01:03,  2.88it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  21% 48/229 [00:15<01:05,  2.77it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  21% 49/229 [00:16<01:06,  2.70it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  22% 50/229 [00:16<01:04,  2.78it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  22% 51/229 [00:17<01:04,  2.78it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  23% 52/229 [00:17<01:02,  2.84it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  23% 53/229 [00:17<01:00,  2.89it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  24% 54/229 [00:18<01:00,  2.92it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  24% 55/229 [00:18<00:57,  3.02it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  24% 56/229 [00:18<00:57,  3.01it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  25% 57/229 [00:18<00:57,  3.01it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  25% 58/229 [00:19<00:55,  3.09it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  26% 59/229 [00:19<00:55,  3.07it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  26% 60/229 [00:19<00:55,  3.05it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  27% 61/229 [00:20<00:51,  3.24it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  27% 62/229 [00:20<00:52,  3.18it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  28% 63/229 [00:20<00:53,  3.12it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  28% 64/229 [00:21<00:52,  3.16it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  28% 65/229 [00:21<00:52,  3.12it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  29% 66/229 [00:21<00:52,  3.08it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  29% 67/229 [00:22<00:54,  2.98it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  30% 68/229 [00:22<00:53,  2.99it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  30% 69/229 [00:22<00:53,  3.00it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  31% 70/229 [00:23<00:53,  3.00it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  31% 71/229 [00:23<00:52,  3.00it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  31% 72/229 [00:23<00:52,  3.00it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  32% 73/229 [00:24<00:51,  3.01it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  32% 74/229 [00:24<00:51,  3.01it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  33% 75/229 [00:24<00:51,  3.02it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  33% 76/229 [00:25<00:50,  3.02it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  34% 77/229 [00:25<00:50,  3.01it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  34% 78/229 [00:25<00:50,  3.01it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  34% 79/229 [00:26<00:48,  3.09it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  35% 80/229 [00:26<00:49,  2.98it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  35% 81/229 [00:26<00:48,  3.07it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  36% 82/229 [00:27<00:48,  3.05it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  36% 83/229 [00:27<00:46,  3.11it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  37% 84/229 [00:27<00:45,  3.17it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  37% 85/229 [00:28<00:45,  3.20it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  38% 86/229 [00:28<00:45,  3.14it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  38% 87/229 [00:28<00:45,  3.10it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  38% 88/229 [00:29<00:46,  3.00it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  39% 89/229 [00:29<00:45,  3.09it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  39% 90/229 [00:29<00:46,  2.98it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  40% 91/229 [00:30<00:46,  3.00it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  40% 92/229 [00:30<00:44,  3.08it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  41% 93/229 [00:30<00:44,  3.05it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  41% 94/229 [00:31<00:44,  3.04it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  41% 95/229 [00:31<00:44,  3.03it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  42% 96/229 [00:31<00:43,  3.03it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  42% 97/229 [00:32<00:43,  3.02it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  43% 98/229 [00:32<00:43,  3.03it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  43% 99/229 [00:32<00:42,  3.03it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  44% 100/229 [00:33<00:41,  3.11it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  44% 101/229 [00:33<00:40,  3.16it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  45% 102/229 [00:33<00:40,  3.10it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  45% 103/229 [00:34<00:42,  2.99it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  45% 104/229 [00:34<00:42,  2.93it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  46% 105/229 [00:34<00:42,  2.95it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  46% 106/229 [00:35<00:41,  2.97it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  47% 107/229 [00:35<00:41,  2.97it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  47% 108/229 [00:35<00:40,  2.98it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  48% 109/229 [00:36<00:39,  3.06it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  48% 110/229 [00:36<00:38,  3.06it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  48% 111/229 [00:36<00:39,  2.96it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  49% 112/229 [00:37<00:38,  3.06it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  49% 113/229 [00:37<00:37,  3.13it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  50% 114/229 [00:37<00:37,  3.10it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  50% 115/229 [00:38<00:38,  2.99it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  51% 116/229 [00:38<00:37,  2.99it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  51% 117/229 [00:38<00:38,  2.92it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  52% 118/229 [00:39<00:38,  2.87it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  52% 119/229 [00:39<00:36,  2.99it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  52% 120/229 [00:39<00:36,  3.00it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  53% 121/229 [00:40<00:35,  3.01it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  53% 122/229 [00:40<00:35,  3.02it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  54% 123/229 [00:40<00:35,  3.02it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  54% 124/229 [00:41<00:33,  3.10it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  55% 125/229 [00:41<00:35,  2.90it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  55% 126/229 [00:41<00:35,  2.86it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  55% 127/229 [00:42<00:35,  2.91it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  56% 128/229 [00:42<00:35,  2.86it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  56% 129/229 [00:42<00:34,  2.92it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  57% 130/229 [00:43<00:33,  2.94it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  57% 131/229 [00:43<00:34,  2.82it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  58% 132/229 [00:43<00:33,  2.87it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  58% 133/229 [00:44<00:32,  2.92it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  59% 134/229 [00:44<00:32,  2.95it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  59% 135/229 [00:44<00:31,  2.97it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  59% 136/229 [00:45<00:30,  3.06it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  60% 137/229 [00:45<00:30,  2.97it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  60% 138/229 [00:45<00:29,  3.06it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  61% 139/229 [00:46<00:29,  3.04it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  61% 140/229 [00:46<00:28,  3.11it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  62% 141/229 [00:46<00:27,  3.17it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  62% 142/229 [00:47<00:27,  3.12it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  62% 143/229 [00:47<00:28,  3.00it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  63% 144/229 [00:47<00:28,  2.93it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  63% 145/229 [00:48<00:30,  2.71it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  64% 146/229 [00:48<00:30,  2.73it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  64% 147/229 [00:48<00:29,  2.81it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  65% 148/229 [00:49<00:28,  2.87it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  65% 149/229 [00:49<00:28,  2.84it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  66% 150/229 [00:49<00:26,  2.97it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  66% 151/229 [00:50<00:26,  2.99it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  66% 152/229 [00:50<00:25,  3.00it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  67% 153/229 [00:50<00:24,  3.08it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  67% 154/229 [00:51<00:24,  3.06it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  68% 155/229 [00:51<00:24,  3.04it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  68% 156/229 [00:51<00:24,  3.03it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  69% 157/229 [00:52<00:23,  3.03it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  69% 158/229 [00:52<00:23,  3.02it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  69% 159/229 [00:52<00:22,  3.10it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  70% 160/229 [00:53<00:21,  3.15it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  70% 161/229 [00:53<00:21,  3.11it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  71% 162/229 [00:53<00:21,  3.16it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  71% 163/229 [00:54<00:21,  3.11it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  72% 164/229 [00:54<00:21,  2.98it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  72% 165/229 [00:54<00:20,  3.07it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  72% 166/229 [00:55<00:20,  3.04it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  73% 167/229 [00:55<00:19,  3.11it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  73% 168/229 [00:55<00:19,  3.08it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  74% 169/229 [00:56<00:19,  3.14it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  74% 170/229 [00:56<00:19,  3.02it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  75% 171/229 [00:56<00:18,  3.10it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  75% 172/229 [00:57<00:18,  3.08it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  76% 173/229 [00:57<00:17,  3.14it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  76% 174/229 [00:57<00:17,  3.11it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  76% 175/229 [00:57<00:17,  3.16it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  77% 176/229 [00:58<00:15,  3.34it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  77% 177/229 [00:58<00:16,  3.24it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  78% 178/229 [00:58<00:16,  3.08it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  78% 179/229 [00:59<00:15,  3.15it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  79% 180/229 [00:59<00:15,  3.19it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  79% 181/229 [00:59<00:15,  3.13it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  79% 182/229 [01:00<00:14,  3.17it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  80% 183/229 [01:00<00:14,  3.21it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  80% 184/229 [01:00<00:13,  3.24it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  81% 185/229 [01:01<00:13,  3.17it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  81% 186/229 [01:01<00:13,  3.23it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  82% 187/229 [01:01<00:13,  3.17it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  82% 188/229 [01:02<00:13,  3.04it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  83% 189/229 [01:02<00:13,  2.95it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  83% 190/229 [01:02<00:13,  2.98it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  83% 191/229 [01:03<00:12,  2.99it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  84% 192/229 [01:03<00:11,  3.10it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  84% 193/229 [01:03<00:11,  3.16it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  85% 194/229 [01:04<00:10,  3.20it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  85% 195/229 [01:04<00:10,  3.15it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  86% 196/229 [01:04<00:10,  3.19it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  86% 197/229 [01:05<00:10,  3.06it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  86% 198/229 [01:05<00:09,  3.13it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  87% 199/229 [01:05<00:09,  3.18it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  87% 200/229 [01:05<00:09,  3.14it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  88% 201/229 [01:06<00:09,  3.10it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  88% 202/229 [01:06<00:09,  2.91it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  89% 203/229 [01:06<00:08,  3.02it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  89% 204/229 [01:07<00:08,  3.02it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  90% 205/229 [01:07<00:07,  3.10it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  90% 206/229 [01:07<00:07,  2.99it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  90% 207/229 [01:08<00:07,  3.00it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  91% 208/229 [01:08<00:06,  3.08it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  91% 209/229 [01:08<00:06,  3.14it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  92% 210/229 [01:09<00:06,  3.10it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  92% 211/229 [01:09<00:05,  3.07it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  93% 212/229 [01:09<00:05,  3.13it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  93% 213/229 [01:10<00:05,  3.02it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  93% 214/229 [01:10<00:04,  3.01it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  94% 215/229 [01:10<00:04,  2.93it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  94% 216/229 [01:11<00:04,  3.03it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  95% 217/229 [01:11<00:04,  2.94it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  95% 218/229 [01:11<00:03,  2.97it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  96% 219/229 [01:12<00:03,  2.98it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  96% 220/229 [01:12<00:02,  3.01it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  97% 221/229 [01:12<00:02,  3.01it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  97% 222/229 [01:13<00:02,  2.95it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  97% 223/229 [01:13<00:01,  3.05it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  98% 224/229 [01:13<00:01,  2.97it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  98% 225/229 [01:14<00:01,  2.91it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  99% 226/229 [01:14<00:00,  3.03it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset:  99% 227/229 [01:14<00:00,  3.12it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset: 100% 228/229 [01:15<00:00,  3.19it/s]\u001b[A\n",
            "epoch 003 | valid on 'valid' subset: 100% 229/229 [01:15<00:00,  3.97it/s]\u001b[A\n",
            "                                                                          \u001b[A2021-02-28 17:12:18 | INFO | valid | epoch 003 | valid on 'valid' subset | loss 0.506 | nll_loss 0.017 | accuracy 86.4 | wps 707 | wpb 232.7 | bsz 8 | num_updates 429 | best_accuracy 90.6\n",
            "2021-02-28 17:12:18 | INFO | fairseq_cli.train | begin save checkpoint\n",
            "2021-02-28 17:15:53 | INFO | fairseq.checkpoint_utils | saved checkpoint checkpoints/checkpoint3.pt (epoch 3 @ 429 updates, score 86.4) (writing took 214.7876592140001 seconds)\n",
            "2021-02-28 17:15:53 | INFO | fairseq_cli.train | end of epoch 3 (average epoch stats below)\n",
            "2021-02-28 17:15:53 | INFO | train | epoch 003 | loss 0.21 | nll_loss 0.007 | accuracy 95.2 | wps 158.3 | ups 0.17 | wpb 957.5 | bsz 32 | num_updates 429 | lr 9.37592e-06 | gnorm 16.54 | loss_scale 32 | train_wall 574 | wall 2726\n",
            "epoch 004:   0% 0/143 [00:00<?, ?it/s]2021-02-28 17:15:53 | INFO | fairseq.trainer | begin training epoch 4\n",
            "epoch 004:  99% 142/143 [09:30<00:03,  3.94s/it, loss=0.182, nll_loss=0.006, accuracy=95.8, wps=137.8, ups=0.14, wpb=952.9, bsz=32, num_updates=500, lr=9.16691e-06, gnorm=20.508, loss_scale=32, train_wall=401, wall=3012]2021-02-28 17:25:28 | INFO | fairseq_cli.train | begin validation on \"valid\" subset\n",
            "\n",
            "epoch 004 | valid on 'valid' subset:   0% 0/229 [00:00<?, ?it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:   0% 1/229 [00:00<01:26,  2.63it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:   1% 2/229 [00:00<01:24,  2.68it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:   1% 3/229 [00:01<01:19,  2.83it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:   2% 4/229 [00:01<01:17,  2.89it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:   2% 5/229 [00:01<01:16,  2.94it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:   3% 6/229 [00:02<01:15,  2.96it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:   3% 7/229 [00:02<01:14,  2.98it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:   3% 8/229 [00:02<01:12,  3.07it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:   4% 9/229 [00:02<01:12,  3.05it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:   4% 10/229 [00:03<01:10,  3.12it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:   5% 11/229 [00:03<01:12,  3.01it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:   5% 12/229 [00:03<01:12,  3.00it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:   6% 13/229 [00:04<01:10,  3.08it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:   6% 14/229 [00:04<01:10,  3.05it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:   7% 15/229 [00:04<01:10,  3.04it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:   7% 16/229 [00:05<01:10,  3.03it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:   7% 17/229 [00:05<01:08,  3.10it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:   8% 18/229 [00:05<01:10,  2.99it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:   8% 19/229 [00:06<01:11,  2.92it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:   9% 20/229 [00:06<01:09,  3.02it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:   9% 21/229 [00:06<01:09,  3.01it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  10% 22/229 [00:07<01:06,  3.10it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  10% 23/229 [00:07<01:05,  3.16it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  10% 24/229 [00:07<01:05,  3.11it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  11% 25/229 [00:08<01:06,  3.09it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  11% 26/229 [00:08<01:06,  3.07it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  12% 27/229 [00:08<01:04,  3.14it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  12% 28/229 [00:09<01:04,  3.11it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  13% 29/229 [00:09<01:04,  3.08it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  13% 30/229 [00:09<01:03,  3.15it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  14% 31/229 [00:10<01:03,  3.11it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  14% 32/229 [00:10<01:03,  3.08it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  14% 33/229 [00:10<01:02,  3.13it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  15% 34/229 [00:11<01:04,  3.01it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  15% 35/229 [00:11<01:06,  2.93it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  16% 36/229 [00:11<01:03,  3.04it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  16% 37/229 [00:12<01:03,  3.03it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  17% 38/229 [00:12<01:03,  3.03it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  17% 39/229 [00:12<01:02,  3.03it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  17% 40/229 [00:13<01:00,  3.11it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  18% 41/229 [00:13<01:01,  3.08it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  18% 42/229 [00:13<01:02,  2.98it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  19% 43/229 [00:14<01:05,  2.84it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  19% 44/229 [00:14<01:04,  2.89it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  20% 45/229 [00:14<01:01,  3.00it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  20% 46/229 [00:15<01:02,  2.93it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  21% 47/229 [00:15<01:03,  2.89it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  21% 48/229 [00:15<01:05,  2.77it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  21% 49/229 [00:16<01:06,  2.70it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  22% 50/229 [00:16<01:04,  2.78it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  22% 51/229 [00:17<01:03,  2.78it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  23% 52/229 [00:17<01:02,  2.84it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  23% 53/229 [00:17<01:00,  2.89it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  24% 54/229 [00:18<00:59,  2.92it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  24% 55/229 [00:18<00:57,  3.02it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  24% 56/229 [00:18<00:57,  3.01it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  25% 57/229 [00:19<00:57,  3.01it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  25% 58/229 [00:19<00:55,  3.09it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  26% 59/229 [00:19<00:55,  3.06it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  26% 60/229 [00:19<00:55,  3.04it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  27% 61/229 [00:20<00:51,  3.24it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  27% 62/229 [00:20<00:52,  3.17it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  28% 63/229 [00:20<00:53,  3.11it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  28% 64/229 [00:21<00:52,  3.16it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  28% 65/229 [00:21<00:52,  3.12it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  29% 66/229 [00:21<00:52,  3.08it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  29% 67/229 [00:22<00:54,  2.99it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  30% 68/229 [00:22<00:53,  2.99it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  30% 69/229 [00:22<00:53,  3.00it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  31% 70/229 [00:23<00:53,  2.99it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  31% 71/229 [00:23<00:52,  3.00it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  31% 72/229 [00:23<00:52,  3.00it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  32% 73/229 [00:24<00:51,  3.01it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  32% 74/229 [00:24<00:51,  3.00it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  33% 75/229 [00:24<00:51,  3.00it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  33% 76/229 [00:25<00:50,  3.00it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  34% 77/229 [00:25<00:50,  3.00it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  34% 78/229 [00:25<00:50,  3.01it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  34% 79/229 [00:26<00:48,  3.08it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  35% 80/229 [00:26<00:49,  2.98it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  35% 81/229 [00:26<00:48,  3.06it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  36% 82/229 [00:27<00:48,  3.05it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  36% 83/229 [00:27<00:46,  3.11it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  37% 84/229 [00:27<00:45,  3.16it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  37% 85/229 [00:28<00:45,  3.20it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  38% 86/229 [00:28<00:45,  3.14it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  38% 87/229 [00:28<00:45,  3.11it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  38% 88/229 [00:29<00:47,  3.00it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  39% 89/229 [00:29<00:45,  3.08it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  39% 90/229 [00:29<00:46,  2.98it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  40% 91/229 [00:30<00:46,  2.99it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  40% 92/229 [00:30<00:44,  3.07it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  41% 93/229 [00:30<00:44,  3.05it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  41% 94/229 [00:31<00:44,  3.03it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  41% 95/229 [00:31<00:44,  3.02it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  42% 96/229 [00:31<00:43,  3.02it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  42% 97/229 [00:32<00:43,  3.02it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  43% 98/229 [00:32<00:43,  3.03it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  43% 99/229 [00:32<00:42,  3.04it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  44% 100/229 [00:33<00:41,  3.10it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  44% 101/229 [00:33<00:40,  3.16it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  45% 102/229 [00:33<00:40,  3.12it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  45% 103/229 [00:34<00:41,  3.00it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  45% 104/229 [00:34<00:42,  2.93it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  46% 105/229 [00:34<00:41,  2.95it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  46% 106/229 [00:35<00:41,  2.97it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  47% 107/229 [00:35<00:40,  2.98it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  47% 108/229 [00:35<00:40,  2.98it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  48% 109/229 [00:36<00:39,  3.07it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  48% 110/229 [00:36<00:38,  3.06it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  48% 111/229 [00:36<00:39,  2.95it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  49% 112/229 [00:37<00:38,  3.05it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  49% 113/229 [00:37<00:37,  3.12it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  50% 114/229 [00:37<00:37,  3.08it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  50% 115/229 [00:38<00:38,  2.97it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  51% 116/229 [00:38<00:37,  2.98it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  51% 117/229 [00:38<00:38,  2.90it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  52% 118/229 [00:39<00:38,  2.86it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  52% 119/229 [00:39<00:36,  2.97it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  52% 120/229 [00:39<00:36,  2.98it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  53% 121/229 [00:40<00:36,  3.00it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  53% 122/229 [00:40<00:35,  3.00it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  54% 123/229 [00:40<00:35,  3.01it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  54% 124/229 [00:41<00:33,  3.10it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  55% 125/229 [00:41<00:35,  2.90it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  55% 126/229 [00:41<00:35,  2.86it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  55% 127/229 [00:42<00:35,  2.91it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  56% 128/229 [00:42<00:35,  2.87it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  56% 129/229 [00:42<00:34,  2.92it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  57% 130/229 [00:43<00:33,  2.95it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  57% 131/229 [00:43<00:34,  2.82it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  58% 132/229 [00:43<00:33,  2.88it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  58% 133/229 [00:44<00:32,  2.92it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  59% 134/229 [00:44<00:32,  2.95it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  59% 135/229 [00:44<00:31,  2.97it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  59% 136/229 [00:45<00:30,  3.07it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  60% 137/229 [00:45<00:30,  2.98it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  60% 138/229 [00:45<00:29,  3.06it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  61% 139/229 [00:46<00:29,  3.04it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  61% 140/229 [00:46<00:28,  3.11it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  62% 141/229 [00:46<00:27,  3.16it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  62% 142/229 [00:47<00:27,  3.12it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  62% 143/229 [00:47<00:28,  3.00it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  63% 144/229 [00:47<00:28,  2.93it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  63% 145/229 [00:48<00:30,  2.72it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  64% 146/229 [00:48<00:30,  2.74it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  64% 147/229 [00:48<00:29,  2.81it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  65% 148/229 [00:49<00:28,  2.87it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  65% 149/229 [00:49<00:28,  2.84it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  66% 150/229 [00:49<00:26,  2.96it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  66% 151/229 [00:50<00:26,  2.97it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  66% 152/229 [00:50<00:25,  2.98it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  67% 153/229 [00:50<00:24,  3.07it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  67% 154/229 [00:51<00:24,  3.05it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  68% 155/229 [00:51<00:24,  3.04it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  68% 156/229 [00:51<00:24,  3.03it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  69% 157/229 [00:52<00:23,  3.02it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  69% 158/229 [00:52<00:23,  3.02it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  69% 159/229 [00:52<00:22,  3.10it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  70% 160/229 [00:53<00:21,  3.15it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  70% 161/229 [00:53<00:21,  3.11it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  71% 162/229 [00:53<00:21,  3.17it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  71% 163/229 [00:54<00:21,  3.12it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  72% 164/229 [00:54<00:21,  3.00it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  72% 165/229 [00:54<00:20,  3.08it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  72% 166/229 [00:55<00:20,  3.06it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  73% 167/229 [00:55<00:19,  3.13it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  73% 168/229 [00:55<00:19,  3.09it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  74% 169/229 [00:56<00:19,  3.15it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  74% 170/229 [00:56<00:19,  3.03it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  75% 171/229 [00:56<00:18,  3.10it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  75% 172/229 [00:57<00:18,  3.08it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  76% 173/229 [00:57<00:17,  3.15it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  76% 174/229 [00:57<00:17,  3.11it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  76% 175/229 [00:58<00:17,  3.16it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  77% 176/229 [00:58<00:15,  3.33it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  77% 177/229 [00:58<00:16,  3.22it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  78% 178/229 [00:58<00:16,  3.07it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  78% 179/229 [00:59<00:15,  3.14it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  79% 180/229 [00:59<00:15,  3.18it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  79% 181/229 [00:59<00:15,  3.13it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  79% 182/229 [01:00<00:14,  3.18it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  80% 183/229 [01:00<00:14,  3.21it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  80% 184/229 [01:00<00:13,  3.24it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  81% 185/229 [01:01<00:13,  3.17it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  81% 186/229 [01:01<00:13,  3.21it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  82% 187/229 [01:01<00:13,  3.14it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  82% 188/229 [01:02<00:13,  3.02it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  83% 189/229 [01:02<00:13,  2.93it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  83% 190/229 [01:02<00:13,  2.96it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  83% 191/229 [01:03<00:12,  2.97it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  84% 192/229 [01:03<00:11,  3.09it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  84% 193/229 [01:03<00:11,  3.14it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  85% 194/229 [01:04<00:10,  3.18it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  85% 195/229 [01:04<00:10,  3.14it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  86% 196/229 [01:04<00:10,  3.18it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  86% 197/229 [01:05<00:10,  3.04it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  86% 198/229 [01:05<00:09,  3.11it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  87% 199/229 [01:05<00:09,  3.16it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  87% 200/229 [01:06<00:09,  3.12it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  88% 201/229 [01:06<00:09,  3.09it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  88% 202/229 [01:06<00:09,  2.89it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  89% 203/229 [01:07<00:08,  3.01it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  89% 204/229 [01:07<00:08,  3.01it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  90% 205/229 [01:07<00:07,  3.09it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  90% 206/229 [01:08<00:07,  2.98it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  90% 207/229 [01:08<00:07,  2.99it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  91% 208/229 [01:08<00:06,  3.08it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  91% 209/229 [01:08<00:06,  3.14it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  92% 210/229 [01:09<00:06,  3.11it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  92% 211/229 [01:09<00:05,  3.08it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  93% 212/229 [01:09<00:05,  3.14it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  93% 213/229 [01:10<00:05,  3.02it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  93% 214/229 [01:10<00:04,  3.01it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  94% 215/229 [01:11<00:04,  2.93it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  94% 216/229 [01:11<00:04,  3.03it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  95% 217/229 [01:11<00:04,  2.94it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  95% 218/229 [01:12<00:03,  2.97it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  96% 219/229 [01:12<00:03,  2.98it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  96% 220/229 [01:12<00:03,  3.00it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  97% 221/229 [01:12<00:02,  3.01it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  97% 222/229 [01:13<00:02,  2.95it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  97% 223/229 [01:13<00:01,  3.05it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  98% 224/229 [01:14<00:01,  2.96it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  98% 225/229 [01:14<00:01,  2.91it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  99% 226/229 [01:14<00:00,  3.03it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset:  99% 227/229 [01:14<00:00,  3.11it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset: 100% 228/229 [01:15<00:00,  3.19it/s]\u001b[A\n",
            "epoch 004 | valid on 'valid' subset: 100% 229/229 [01:15<00:00,  3.97it/s]\u001b[A\n",
            "                                                                          \u001b[A2021-02-28 17:26:43 | INFO | valid | epoch 004 | valid on 'valid' subset | loss 0.36 | nll_loss 0.012 | accuracy 92.6 | wps 706.4 | wpb 232.7 | bsz 8 | num_updates 572 | best_accuracy 92.6\n",
            "2021-02-28 17:26:43 | INFO | fairseq_cli.train | begin save checkpoint\n",
            "2021-02-28 17:32:25 | INFO | fairseq.checkpoint_utils | saved checkpoint checkpoints/checkpoint4.pt (epoch 4 @ 572 updates, score 92.6) (writing took 341.53966797799967 seconds)\n",
            "2021-02-28 17:32:25 | INFO | fairseq_cli.train | end of epoch 4 (average epoch stats below)\n",
            "2021-02-28 17:32:25 | INFO | train | epoch 004 | loss 0.149 | nll_loss 0.005 | accuracy 96.5 | wps 138 | ups 0.14 | wpb 957.5 | bsz 32 | num_updates 572 | lr 8.95496e-06 | gnorm 18.026 | loss_scale 64 | train_wall 574 | wall 3718\n",
            "epoch 005:   0% 0/143 [00:00<?, ?it/s]2021-02-28 17:32:25 | INFO | fairseq.trainer | begin training epoch 5\n",
            "epoch 005:  99% 142/143 [09:30<00:03,  3.98s/it, loss=0.084, nll_loss=0.003, accuracy=98.2, wps=237.6, ups=0.25, wpb=953.8, bsz=32, num_updates=700, lr=8.57816e-06, gnorm=10.711, loss_scale=128, train_wall=401, wall=4233]2021-02-28 17:42:00 | INFO | fairseq_cli.train | begin validation on \"valid\" subset\n",
            "\n",
            "epoch 005 | valid on 'valid' subset:   0% 0/229 [00:00<?, ?it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:   0% 1/229 [00:00<01:26,  2.63it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:   1% 2/229 [00:00<01:24,  2.68it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:   1% 3/229 [00:01<01:19,  2.84it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:   2% 4/229 [00:01<01:17,  2.89it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:   2% 5/229 [00:01<01:16,  2.94it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:   3% 6/229 [00:02<01:15,  2.96it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:   3% 7/229 [00:02<01:14,  2.98it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:   3% 8/229 [00:02<01:11,  3.07it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:   4% 9/229 [00:02<01:11,  3.06it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:   4% 10/229 [00:03<01:10,  3.12it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:   5% 11/229 [00:03<01:12,  3.01it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:   5% 12/229 [00:03<01:12,  3.00it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:   6% 13/229 [00:04<01:10,  3.08it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:   6% 14/229 [00:04<01:10,  3.06it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:   7% 15/229 [00:04<01:10,  3.04it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:   7% 16/229 [00:05<01:10,  3.04it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:   7% 17/229 [00:05<01:08,  3.10it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:   8% 18/229 [00:05<01:10,  2.98it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:   8% 19/229 [00:06<01:12,  2.91it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:   9% 20/229 [00:06<01:09,  3.02it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:   9% 21/229 [00:06<01:09,  3.01it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  10% 22/229 [00:07<01:06,  3.09it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  10% 23/229 [00:07<01:05,  3.15it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  10% 24/229 [00:07<01:06,  3.10it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  11% 25/229 [00:08<01:06,  3.07it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  11% 26/229 [00:08<01:06,  3.05it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  12% 27/229 [00:08<01:04,  3.12it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  12% 28/229 [00:09<01:04,  3.09it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  13% 29/229 [00:09<01:05,  3.06it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  13% 30/229 [00:09<01:03,  3.13it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  14% 31/229 [00:10<01:04,  3.09it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  14% 32/229 [00:10<01:04,  3.06it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  14% 33/229 [00:10<01:02,  3.13it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  15% 34/229 [00:11<01:04,  3.01it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  15% 35/229 [00:11<01:06,  2.94it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  16% 36/229 [00:11<01:03,  3.05it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  16% 37/229 [00:12<01:03,  3.04it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  17% 38/229 [00:12<01:03,  3.03it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  17% 39/229 [00:12<01:02,  3.02it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  17% 40/229 [00:13<01:01,  3.10it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  18% 41/229 [00:13<01:01,  3.07it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  18% 42/229 [00:13<01:02,  2.98it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  19% 43/229 [00:14<01:05,  2.85it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  19% 44/229 [00:14<01:04,  2.89it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  20% 45/229 [00:14<01:01,  3.00it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  20% 46/229 [00:15<01:02,  2.93it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  21% 47/229 [00:15<01:02,  2.89it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  21% 48/229 [00:15<01:05,  2.78it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  21% 49/229 [00:16<01:06,  2.70it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  22% 50/229 [00:16<01:04,  2.79it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  22% 51/229 [00:17<01:03,  2.79it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  23% 52/229 [00:17<01:02,  2.85it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  23% 53/229 [00:17<01:00,  2.90it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  24% 54/229 [00:18<00:59,  2.93it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  24% 55/229 [00:18<00:57,  3.03it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  24% 56/229 [00:18<00:57,  3.01it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  25% 57/229 [00:19<00:57,  3.01it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  25% 58/229 [00:19<00:55,  3.09it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  26% 59/229 [00:19<00:55,  3.07it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  26% 60/229 [00:19<00:55,  3.04it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  27% 61/229 [00:20<00:51,  3.24it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  27% 62/229 [00:20<00:52,  3.17it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  28% 63/229 [00:20<00:53,  3.12it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  28% 64/229 [00:21<00:52,  3.17it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  28% 65/229 [00:21<00:52,  3.12it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  29% 66/229 [00:21<00:52,  3.09it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  29% 67/229 [00:22<00:54,  2.99it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  30% 68/229 [00:22<00:53,  2.99it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  30% 69/229 [00:22<00:53,  2.99it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  31% 70/229 [00:23<00:53,  2.99it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  31% 71/229 [00:23<00:52,  2.99it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  31% 72/229 [00:23<00:52,  3.00it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  32% 73/229 [00:24<00:51,  3.01it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  32% 74/229 [00:24<00:51,  3.00it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  33% 75/229 [00:24<00:51,  3.01it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  33% 76/229 [00:25<00:50,  3.02it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  34% 77/229 [00:25<00:50,  3.02it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  34% 78/229 [00:25<00:50,  3.02it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  34% 79/229 [00:26<00:48,  3.10it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  35% 80/229 [00:26<00:49,  3.00it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  35% 81/229 [00:26<00:48,  3.08it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  36% 82/229 [00:27<00:48,  3.06it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  36% 83/229 [00:27<00:46,  3.13it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  37% 84/229 [00:27<00:45,  3.18it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  37% 85/229 [00:28<00:44,  3.21it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  38% 86/229 [00:28<00:45,  3.15it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  38% 87/229 [00:28<00:45,  3.12it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  38% 88/229 [00:29<00:46,  3.01it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  39% 89/229 [00:29<00:45,  3.10it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  39% 90/229 [00:29<00:46,  2.99it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  40% 91/229 [00:30<00:45,  3.01it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  40% 92/229 [00:30<00:44,  3.10it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  41% 93/229 [00:30<00:44,  3.07it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  41% 94/229 [00:31<00:44,  3.05it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  41% 95/229 [00:31<00:44,  3.04it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  42% 96/229 [00:31<00:43,  3.04it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  42% 97/229 [00:32<00:43,  3.03it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  43% 98/229 [00:32<00:43,  3.04it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  43% 99/229 [00:32<00:42,  3.04it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  44% 100/229 [00:33<00:41,  3.11it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  44% 101/229 [00:33<00:40,  3.17it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  45% 102/229 [00:33<00:40,  3.12it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  45% 103/229 [00:34<00:41,  3.01it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  45% 104/229 [00:34<00:42,  2.94it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  46% 105/229 [00:34<00:41,  2.96it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  46% 106/229 [00:35<00:41,  2.98it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  47% 107/229 [00:35<00:40,  2.99it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  47% 108/229 [00:35<00:40,  3.00it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  48% 109/229 [00:36<00:38,  3.08it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  48% 110/229 [00:36<00:38,  3.07it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  48% 111/229 [00:36<00:39,  2.97it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  49% 112/229 [00:36<00:38,  3.07it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  49% 113/229 [00:37<00:37,  3.13it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  50% 114/229 [00:37<00:37,  3.10it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  50% 115/229 [00:37<00:38,  2.99it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  51% 116/229 [00:38<00:37,  3.00it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  51% 117/229 [00:38<00:38,  2.92it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  52% 118/229 [00:39<00:38,  2.87it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  52% 119/229 [00:39<00:36,  3.00it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  52% 120/229 [00:39<00:36,  3.01it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  53% 121/229 [00:40<00:35,  3.02it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  53% 122/229 [00:40<00:35,  3.02it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  54% 123/229 [00:40<00:35,  3.02it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  54% 124/229 [00:40<00:33,  3.10it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  55% 125/229 [00:41<00:35,  2.91it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  55% 126/229 [00:41<00:35,  2.87it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  55% 127/229 [00:42<00:35,  2.91it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  56% 128/229 [00:42<00:35,  2.87it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  56% 129/229 [00:42<00:34,  2.92it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  57% 130/229 [00:43<00:33,  2.94it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  57% 131/229 [00:43<00:34,  2.81it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  58% 132/229 [00:43<00:33,  2.87it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  58% 133/229 [00:44<00:32,  2.92it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  59% 134/229 [00:44<00:32,  2.95it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  59% 135/229 [00:44<00:31,  2.97it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  59% 136/229 [00:45<00:30,  3.07it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  60% 137/229 [00:45<00:30,  2.98it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  60% 138/229 [00:45<00:29,  3.07it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  61% 139/229 [00:46<00:29,  3.04it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  61% 140/229 [00:46<00:28,  3.11it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  62% 141/229 [00:46<00:27,  3.17it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  62% 142/229 [00:47<00:27,  3.12it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  62% 143/229 [00:47<00:28,  3.00it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  63% 144/229 [00:47<00:28,  2.93it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  63% 145/229 [00:48<00:30,  2.71it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  64% 146/229 [00:48<00:30,  2.73it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  64% 147/229 [00:48<00:29,  2.81it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  65% 148/229 [00:49<00:28,  2.87it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  65% 149/229 [00:49<00:28,  2.85it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  66% 150/229 [00:49<00:26,  2.96it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  66% 151/229 [00:50<00:26,  2.98it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  66% 152/229 [00:50<00:25,  2.99it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  67% 153/229 [00:50<00:24,  3.08it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  67% 154/229 [00:51<00:24,  3.06it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  68% 155/229 [00:51<00:24,  3.05it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  68% 156/229 [00:51<00:24,  3.03it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  69% 157/229 [00:52<00:23,  3.03it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  69% 158/229 [00:52<00:23,  3.01it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  69% 159/229 [00:52<00:22,  3.10it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  70% 160/229 [00:53<00:21,  3.16it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  70% 161/229 [00:53<00:21,  3.12it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  71% 162/229 [00:53<00:21,  3.17it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  71% 163/229 [00:54<00:21,  3.13it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  72% 164/229 [00:54<00:21,  3.00it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  72% 165/229 [00:54<00:20,  3.09it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  72% 166/229 [00:55<00:20,  3.08it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  73% 167/229 [00:55<00:19,  3.14it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  73% 168/229 [00:55<00:19,  3.11it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  74% 169/229 [00:55<00:18,  3.16it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  74% 170/229 [00:56<00:19,  3.04it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  75% 171/229 [00:56<00:18,  3.11it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  75% 172/229 [00:56<00:18,  3.09it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  76% 173/229 [00:57<00:17,  3.15it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  76% 174/229 [00:57<00:17,  3.11it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  76% 175/229 [00:57<00:17,  3.16it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  77% 176/229 [00:58<00:15,  3.34it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  77% 177/229 [00:58<00:16,  3.23it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  78% 178/229 [00:58<00:16,  3.08it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  78% 179/229 [00:59<00:15,  3.15it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  79% 180/229 [00:59<00:15,  3.19it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  79% 181/229 [00:59<00:15,  3.13it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  79% 182/229 [01:00<00:14,  3.17it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  80% 183/229 [01:00<00:14,  3.21it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  80% 184/229 [01:00<00:13,  3.24it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  81% 185/229 [01:01<00:13,  3.17it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  81% 186/229 [01:01<00:13,  3.23it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  82% 187/229 [01:01<00:13,  3.16it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  82% 188/229 [01:02<00:13,  3.03it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  83% 189/229 [01:02<00:13,  2.94it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  83% 190/229 [01:02<00:13,  2.97it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  83% 191/229 [01:03<00:12,  2.98it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  84% 192/229 [01:03<00:11,  3.09it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  84% 193/229 [01:03<00:11,  3.14it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  85% 194/229 [01:03<00:10,  3.19it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  85% 195/229 [01:04<00:10,  3.15it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  86% 196/229 [01:04<00:10,  3.19it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  86% 197/229 [01:04<00:10,  3.06it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  86% 198/229 [01:05<00:09,  3.12it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  87% 199/229 [01:05<00:09,  3.16it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  87% 200/229 [01:05<00:09,  3.11it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  88% 201/229 [01:06<00:09,  3.08it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  88% 202/229 [01:06<00:09,  2.89it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  89% 203/229 [01:06<00:08,  3.00it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  89% 204/229 [01:07<00:08,  3.01it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  90% 205/229 [01:07<00:07,  3.08it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  90% 206/229 [01:07<00:07,  2.98it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  90% 207/229 [01:08<00:07,  2.99it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  91% 208/229 [01:08<00:06,  3.07it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  91% 209/229 [01:08<00:06,  3.13it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  92% 210/229 [01:09<00:06,  3.10it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  92% 211/229 [01:09<00:05,  3.06it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  93% 212/229 [01:09<00:05,  3.12it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  93% 213/229 [01:10<00:05,  3.01it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  93% 214/229 [01:10<00:04,  3.01it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  94% 215/229 [01:10<00:04,  2.93it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  94% 216/229 [01:11<00:04,  3.04it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  95% 217/229 [01:11<00:04,  2.95it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  95% 218/229 [01:11<00:03,  2.98it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  96% 219/229 [01:12<00:03,  2.99it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  96% 220/229 [01:12<00:02,  3.01it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  97% 221/229 [01:12<00:02,  3.01it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  97% 222/229 [01:13<00:02,  2.95it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  97% 223/229 [01:13<00:01,  3.05it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  98% 224/229 [01:13<00:01,  2.96it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  98% 225/229 [01:14<00:01,  2.91it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  99% 226/229 [01:14<00:00,  3.03it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset:  99% 227/229 [01:14<00:00,  3.11it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset: 100% 228/229 [01:15<00:00,  3.18it/s]\u001b[A\n",
            "epoch 005 | valid on 'valid' subset: 100% 229/229 [01:15<00:00,  3.95it/s]\u001b[A\n",
            "                                                                          \u001b[A2021-02-28 17:43:15 | INFO | valid | epoch 005 | valid on 'valid' subset | loss 0.679 | nll_loss 0.023 | accuracy 89 | wps 707.3 | wpb 232.7 | bsz 8 | num_updates 715 | best_accuracy 92.6\n",
            "2021-02-28 17:43:15 | INFO | fairseq_cli.train | begin save checkpoint\n",
            "2021-02-28 17:46:48 | INFO | fairseq.checkpoint_utils | saved checkpoint checkpoints/checkpoint5.pt (epoch 5 @ 715 updates, score 89.0) (writing took 213.62495311699968 seconds)\n",
            "2021-02-28 17:46:48 | INFO | fairseq_cli.train | end of epoch 5 (average epoch stats below)\n",
            "2021-02-28 17:46:48 | INFO | train | epoch 005 | loss 0.082 | nll_loss 0.003 | accuracy 98.2 | wps 158.6 | ups 0.17 | wpb 957.5 | bsz 32 | num_updates 715 | lr 8.534e-06 | gnorm 11.204 | loss_scale 128 | train_wall 574 | wall 4582\n",
            "epoch 006:   0% 0/143 [00:00<?, ?it/s]2021-02-28 17:46:49 | INFO | fairseq.trainer | begin training epoch 6\n",
            "epoch 006:  93% 133/143 [08:54<00:40,  4.06s/it, loss=0.059, nll_loss=0.002, accuracy=98.6, wps=139, ups=0.14, wpb=962.3, bsz=32, num_updates=800, lr=8.28378e-06, gnorm=11.161, loss_scale=256, train_wall=403, wall=4925]2021-02-28 17:55:47 | INFO | fairseq.trainer | NOTE: overflow detected, setting loss scale to: 128.0\n",
            "epoch 006:  99% 142/143 [09:30<00:04,  4.00s/it, loss=0.059, nll_loss=0.002, accuracy=98.6, wps=139, ups=0.14, wpb=962.3, bsz=32, num_updates=800, lr=8.28378e-06, gnorm=11.161, loss_scale=256, train_wall=403, wall=4925]2021-02-28 17:56:23 | INFO | fairseq.trainer | NOTE: overflow detected, setting loss scale to: 64.0\n",
            "2021-02-28 17:56:23 | INFO | fairseq_cli.train | begin validation on \"valid\" subset\n",
            "\n",
            "epoch 006 | valid on 'valid' subset:   0% 0/229 [00:00<?, ?it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:   0% 1/229 [00:00<01:26,  2.63it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:   1% 2/229 [00:00<01:24,  2.67it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:   1% 3/229 [00:01<01:19,  2.84it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:   2% 4/229 [00:01<01:17,  2.89it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:   2% 5/229 [00:01<01:16,  2.94it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:   3% 6/229 [00:02<01:15,  2.96it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:   3% 7/229 [00:02<01:14,  2.98it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:   3% 8/229 [00:02<01:12,  3.07it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:   4% 9/229 [00:02<01:12,  3.05it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:   4% 10/229 [00:03<01:10,  3.11it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:   5% 11/229 [00:03<01:12,  3.00it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:   5% 12/229 [00:03<01:12,  3.00it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:   6% 13/229 [00:04<01:10,  3.08it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:   6% 14/229 [00:04<01:10,  3.05it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:   7% 15/229 [00:04<01:10,  3.03it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:   7% 16/229 [00:05<01:10,  3.03it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:   7% 17/229 [00:05<01:08,  3.10it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:   8% 18/229 [00:05<01:10,  2.99it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:   8% 19/229 [00:06<01:11,  2.92it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:   9% 20/229 [00:06<01:09,  3.02it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:   9% 21/229 [00:06<01:08,  3.02it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  10% 22/229 [00:07<01:06,  3.09it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  10% 23/229 [00:07<01:05,  3.14it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  10% 24/229 [00:07<01:06,  3.10it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  11% 25/229 [00:08<01:06,  3.07it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  11% 26/229 [00:08<01:06,  3.05it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  12% 27/229 [00:08<01:04,  3.12it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  12% 28/229 [00:09<01:05,  3.07it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  13% 29/229 [00:09<01:05,  3.05it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  13% 30/229 [00:09<01:03,  3.12it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  14% 31/229 [00:10<01:04,  3.08it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  14% 32/229 [00:10<01:04,  3.06it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  14% 33/229 [00:10<01:02,  3.13it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  15% 34/229 [00:11<01:04,  3.01it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  15% 35/229 [00:11<01:05,  2.94it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  16% 36/229 [00:11<01:03,  3.04it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  16% 37/229 [00:12<01:03,  3.03it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  17% 38/229 [00:12<01:03,  3.03it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  17% 39/229 [00:12<01:02,  3.02it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  17% 40/229 [00:13<01:01,  3.09it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  18% 41/229 [00:13<01:01,  3.06it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  18% 42/229 [00:13<01:03,  2.96it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  19% 43/229 [00:14<01:05,  2.83it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  19% 44/229 [00:14<01:04,  2.87it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  20% 45/229 [00:14<01:01,  2.99it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  20% 46/229 [00:15<01:02,  2.92it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  21% 47/229 [00:15<01:03,  2.88it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  21% 48/229 [00:15<01:05,  2.76it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  21% 49/229 [00:16<01:06,  2.69it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  22% 50/229 [00:16<01:04,  2.78it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  22% 51/229 [00:17<01:03,  2.78it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  23% 52/229 [00:17<01:02,  2.85it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  23% 53/229 [00:17<01:00,  2.90it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  24% 54/229 [00:18<00:59,  2.92it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  24% 55/229 [00:18<00:57,  3.02it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  24% 56/229 [00:18<00:57,  3.02it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  25% 57/229 [00:19<00:57,  3.02it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  25% 58/229 [00:19<00:55,  3.09it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  26% 59/229 [00:19<00:55,  3.07it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  26% 60/229 [00:20<00:55,  3.04it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  27% 61/229 [00:20<00:51,  3.23it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  27% 62/229 [00:20<00:52,  3.16it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  28% 63/229 [00:20<00:53,  3.10it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  28% 64/229 [00:21<00:52,  3.16it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  28% 65/229 [00:21<00:52,  3.11it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  29% 66/229 [00:21<00:52,  3.08it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  29% 67/229 [00:22<00:54,  2.98it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  30% 68/229 [00:22<00:53,  2.99it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  30% 69/229 [00:22<00:53,  3.00it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  31% 70/229 [00:23<00:52,  3.00it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  31% 71/229 [00:23<00:52,  3.01it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  31% 72/229 [00:23<00:52,  3.02it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  32% 73/229 [00:24<00:51,  3.02it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  32% 74/229 [00:24<00:51,  3.02it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  33% 75/229 [00:24<00:51,  3.02it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  33% 76/229 [00:25<00:50,  3.02it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  34% 77/229 [00:25<00:50,  3.01it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  34% 78/229 [00:25<00:50,  3.02it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  34% 79/229 [00:26<00:48,  3.09it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  35% 80/229 [00:26<00:49,  3.00it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  35% 81/229 [00:26<00:48,  3.07it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  36% 82/229 [00:27<00:48,  3.06it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  36% 83/229 [00:27<00:46,  3.12it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  37% 84/229 [00:27<00:45,  3.17it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  37% 85/229 [00:28<00:45,  3.20it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  38% 86/229 [00:28<00:45,  3.14it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  38% 87/229 [00:28<00:45,  3.11it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  38% 88/229 [00:29<00:46,  3.01it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  39% 89/229 [00:29<00:45,  3.08it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  39% 90/229 [00:29<00:46,  2.96it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  40% 91/229 [00:30<00:46,  2.98it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  40% 92/229 [00:30<00:44,  3.06it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  41% 93/229 [00:30<00:44,  3.03it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  41% 94/229 [00:31<00:44,  3.02it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  41% 95/229 [00:31<00:44,  3.00it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  42% 96/229 [00:31<00:44,  3.00it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  42% 97/229 [00:32<00:43,  3.01it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  43% 98/229 [00:32<00:43,  3.01it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  43% 99/229 [00:32<00:43,  3.02it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  44% 100/229 [00:33<00:41,  3.10it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  44% 101/229 [00:33<00:40,  3.15it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  45% 102/229 [00:33<00:41,  3.10it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  45% 103/229 [00:34<00:42,  2.99it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  45% 104/229 [00:34<00:42,  2.92it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  46% 105/229 [00:34<00:42,  2.94it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  46% 106/229 [00:35<00:41,  2.96it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  47% 107/229 [00:35<00:41,  2.97it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  47% 108/229 [00:35<00:40,  2.98it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  48% 109/229 [00:36<00:39,  3.07it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  48% 110/229 [00:36<00:38,  3.07it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  48% 111/229 [00:36<00:39,  2.95it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  49% 112/229 [00:37<00:38,  3.05it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  49% 113/229 [00:37<00:37,  3.12it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  50% 114/229 [00:37<00:37,  3.08it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  50% 115/229 [00:38<00:38,  2.97it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  51% 116/229 [00:38<00:38,  2.97it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  51% 117/229 [00:38<00:38,  2.90it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  52% 118/229 [00:39<00:38,  2.86it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  52% 119/229 [00:39<00:36,  2.98it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  52% 120/229 [00:39<00:36,  2.99it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  53% 121/229 [00:40<00:35,  3.00it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  53% 122/229 [00:40<00:35,  3.01it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  54% 123/229 [00:40<00:35,  3.01it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  54% 124/229 [00:41<00:34,  3.08it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  55% 125/229 [00:41<00:35,  2.90it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  55% 126/229 [00:41<00:36,  2.86it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  55% 127/229 [00:42<00:35,  2.90it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  56% 128/229 [00:42<00:35,  2.85it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  56% 129/229 [00:42<00:34,  2.90it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  57% 130/229 [00:43<00:33,  2.93it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  57% 131/229 [00:43<00:34,  2.80it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  58% 132/229 [00:43<00:33,  2.86it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  58% 133/229 [00:44<00:33,  2.91it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  59% 134/229 [00:44<00:32,  2.94it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  59% 135/229 [00:44<00:31,  2.96it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  59% 136/229 [00:45<00:30,  3.05it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  60% 137/229 [00:45<00:31,  2.96it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  60% 138/229 [00:45<00:29,  3.06it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  61% 139/229 [00:46<00:29,  3.04it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  61% 140/229 [00:46<00:28,  3.11it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  62% 141/229 [00:46<00:27,  3.15it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  62% 142/229 [00:47<00:27,  3.11it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  62% 143/229 [00:47<00:28,  2.98it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  63% 144/229 [00:47<00:29,  2.92it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  63% 145/229 [00:48<00:31,  2.71it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  64% 146/229 [00:48<00:30,  2.73it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  64% 147/229 [00:49<00:29,  2.81it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  65% 148/229 [00:49<00:28,  2.86it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  65% 149/229 [00:49<00:28,  2.84it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  66% 150/229 [00:50<00:26,  2.96it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  66% 151/229 [00:50<00:26,  2.97it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  66% 152/229 [00:50<00:25,  2.99it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  67% 153/229 [00:50<00:24,  3.08it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  67% 154/229 [00:51<00:24,  3.05it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  68% 155/229 [00:51<00:24,  3.05it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  68% 156/229 [00:51<00:24,  3.04it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  69% 157/229 [00:52<00:23,  3.04it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  69% 158/229 [00:52<00:23,  3.03it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  69% 159/229 [00:52<00:22,  3.11it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  70% 160/229 [00:53<00:21,  3.16it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  70% 161/229 [00:53<00:21,  3.12it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  71% 162/229 [00:53<00:21,  3.18it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  71% 163/229 [00:54<00:21,  3.13it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  72% 164/229 [00:54<00:21,  3.01it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  72% 165/229 [00:54<00:20,  3.09it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  72% 166/229 [00:55<00:20,  3.07it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  73% 167/229 [00:55<00:19,  3.14it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  73% 168/229 [00:55<00:19,  3.10it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  74% 169/229 [00:56<00:19,  3.15it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  74% 170/229 [00:56<00:19,  3.03it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  75% 171/229 [00:56<00:18,  3.11it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  75% 172/229 [00:57<00:18,  3.08it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  76% 173/229 [00:57<00:17,  3.14it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  76% 174/229 [00:57<00:17,  3.11it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  76% 175/229 [00:58<00:17,  3.16it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  77% 176/229 [00:58<00:15,  3.33it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  77% 177/229 [00:58<00:16,  3.23it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  78% 178/229 [00:59<00:16,  3.07it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  78% 179/229 [00:59<00:15,  3.14it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  79% 180/229 [00:59<00:15,  3.18it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  79% 181/229 [00:59<00:15,  3.12it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  79% 182/229 [01:00<00:14,  3.16it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  80% 183/229 [01:00<00:14,  3.22it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  80% 184/229 [01:00<00:13,  3.25it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  81% 185/229 [01:01<00:13,  3.18it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  81% 186/229 [01:01<00:13,  3.24it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  82% 187/229 [01:01<00:13,  3.17it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  82% 188/229 [01:02<00:13,  3.04it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  83% 189/229 [01:02<00:13,  2.95it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  83% 190/229 [01:02<00:13,  2.97it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  83% 191/229 [01:03<00:12,  2.98it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  84% 192/229 [01:03<00:11,  3.09it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  84% 193/229 [01:03<00:11,  3.14it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  85% 194/229 [01:04<00:10,  3.19it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  85% 195/229 [01:04<00:10,  3.14it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  86% 196/229 [01:04<00:10,  3.18it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  86% 197/229 [01:05<00:10,  3.04it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  86% 198/229 [01:05<00:09,  3.11it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  87% 199/229 [01:05<00:09,  3.16it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  87% 200/229 [01:06<00:09,  3.11it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  88% 201/229 [01:06<00:09,  3.08it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  88% 202/229 [01:06<00:09,  2.89it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  89% 203/229 [01:07<00:08,  3.00it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  89% 204/229 [01:07<00:08,  3.01it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  90% 205/229 [01:07<00:07,  3.09it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  90% 206/229 [01:08<00:07,  2.98it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  90% 207/229 [01:08<00:07,  2.99it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  91% 208/229 [01:08<00:06,  3.07it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  91% 209/229 [01:09<00:06,  3.14it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  92% 210/229 [01:09<00:06,  3.10it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  92% 211/229 [01:09<00:05,  3.07it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  93% 212/229 [01:10<00:05,  3.13it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  93% 213/229 [01:10<00:05,  3.02it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  93% 214/229 [01:10<00:04,  3.01it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  94% 215/229 [01:11<00:04,  2.93it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  94% 216/229 [01:11<00:04,  3.03it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  95% 217/229 [01:11<00:04,  2.95it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  95% 218/229 [01:12<00:03,  2.97it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  96% 219/229 [01:12<00:03,  2.99it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  96% 220/229 [01:12<00:02,  3.01it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  97% 221/229 [01:13<00:02,  3.02it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  97% 222/229 [01:13<00:02,  2.95it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  97% 223/229 [01:13<00:01,  3.05it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  98% 224/229 [01:14<00:01,  2.96it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  98% 225/229 [01:14<00:01,  2.91it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  99% 226/229 [01:14<00:00,  3.03it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset:  99% 227/229 [01:15<00:00,  3.11it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset: 100% 228/229 [01:15<00:00,  3.18it/s]\u001b[A\n",
            "epoch 006 | valid on 'valid' subset: 100% 229/229 [01:15<00:00,  3.95it/s]\u001b[A\n",
            "                                                                          \u001b[A2021-02-28 17:57:38 | INFO | valid | epoch 006 | valid on 'valid' subset | loss 0.365 | nll_loss 0.012 | accuracy 94.2 | wps 705.9 | wpb 232.7 | bsz 8 | num_updates 856 | best_accuracy 94.2\n",
            "2021-02-28 17:57:38 | INFO | fairseq_cli.train | begin save checkpoint\n",
            "2021-02-28 18:03:20 | INFO | fairseq.checkpoint_utils | saved checkpoint checkpoints/checkpoint6.pt (epoch 6 @ 856 updates, score 94.2) (writing took 341.54576320399974 seconds)\n",
            "2021-02-28 18:03:20 | INFO | fairseq_cli.train | end of epoch 6 (average epoch stats below)\n",
            "2021-02-28 18:03:20 | INFO | train | epoch 006 | loss 0.047 | nll_loss 0.002 | accuracy 98.9 | wps 136.2 | ups 0.14 | wpb 957.9 | bsz 32 | num_updates 856 | lr 8.11893e-06 | gnorm 9.595 | loss_scale 64 | train_wall 574 | wall 5573\n",
            "2021-02-28 18:03:20 | INFO | fairseq_cli.train | done training in 5566.9 seconds\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              ""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9VWprklpka_w"
      },
      "source": [
        "!cp checkpoints/checkpoint_best.pt /content/drive/MyDrive/UPENN/checkpoints/ckpt_6_fin_rob.pt"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nZ12denMpZ9X",
        "outputId": "82f2e90f-9766-42e3-fb4f-f5c232235453"
      },
      "source": [
        "%ls /content/drive/MyDrive/UPENN/checkpoints/"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "ckpt_6_fin_rob.pt\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8NxZQQo0pPbY"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gmGCmGkPYBFc"
      },
      "source": [
        "# Testing the Validation Split"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gml36iwmSprs",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1091e020-b964-455e-bd7b-b6889d6a7f8d"
      },
      "source": [
        "from fairseq.models.roberta import RobertaModel\n",
        "roberta = RobertaModel.from_pretrained(\n",
        "    'checkpoints',\n",
        "    checkpoint_file='checkpoint_best.pt',\n",
        "    data_name_or_path='/content/drive/MyDrive/UPENN/Task1a-bin'\n",
        ")\n",
        "roberta.eval()  # disable dropout"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "RobertaHubInterface(\n",
              "  (model): RobertaModel(\n",
              "    (encoder): RobertaEncoder(\n",
              "      (sentence_encoder): TransformerEncoder(\n",
              "        (dropout_module): FairseqDropout()\n",
              "        (embed_tokens): Embedding(50265, 1024, padding_idx=1)\n",
              "        (embed_positions): LearnedPositionalEmbedding(514, 1024, padding_idx=1)\n",
              "        (layernorm_embedding): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "        (layers): ModuleList(\n",
              "          (0): TransformerEncoderLayer(\n",
              "            (self_attn): MultiheadAttention(\n",
              "              (dropout_module): FairseqDropout()\n",
              "              (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            )\n",
              "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout_module): FairseqDropout()\n",
              "            (activation_dropout_module): FairseqDropout()\n",
              "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          )\n",
              "          (1): TransformerEncoderLayer(\n",
              "            (self_attn): MultiheadAttention(\n",
              "              (dropout_module): FairseqDropout()\n",
              "              (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            )\n",
              "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout_module): FairseqDropout()\n",
              "            (activation_dropout_module): FairseqDropout()\n",
              "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          )\n",
              "          (2): TransformerEncoderLayer(\n",
              "            (self_attn): MultiheadAttention(\n",
              "              (dropout_module): FairseqDropout()\n",
              "              (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            )\n",
              "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout_module): FairseqDropout()\n",
              "            (activation_dropout_module): FairseqDropout()\n",
              "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          )\n",
              "          (3): TransformerEncoderLayer(\n",
              "            (self_attn): MultiheadAttention(\n",
              "              (dropout_module): FairseqDropout()\n",
              "              (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            )\n",
              "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout_module): FairseqDropout()\n",
              "            (activation_dropout_module): FairseqDropout()\n",
              "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          )\n",
              "          (4): TransformerEncoderLayer(\n",
              "            (self_attn): MultiheadAttention(\n",
              "              (dropout_module): FairseqDropout()\n",
              "              (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            )\n",
              "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout_module): FairseqDropout()\n",
              "            (activation_dropout_module): FairseqDropout()\n",
              "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          )\n",
              "          (5): TransformerEncoderLayer(\n",
              "            (self_attn): MultiheadAttention(\n",
              "              (dropout_module): FairseqDropout()\n",
              "              (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            )\n",
              "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout_module): FairseqDropout()\n",
              "            (activation_dropout_module): FairseqDropout()\n",
              "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          )\n",
              "          (6): TransformerEncoderLayer(\n",
              "            (self_attn): MultiheadAttention(\n",
              "              (dropout_module): FairseqDropout()\n",
              "              (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            )\n",
              "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout_module): FairseqDropout()\n",
              "            (activation_dropout_module): FairseqDropout()\n",
              "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          )\n",
              "          (7): TransformerEncoderLayer(\n",
              "            (self_attn): MultiheadAttention(\n",
              "              (dropout_module): FairseqDropout()\n",
              "              (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            )\n",
              "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout_module): FairseqDropout()\n",
              "            (activation_dropout_module): FairseqDropout()\n",
              "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          )\n",
              "          (8): TransformerEncoderLayer(\n",
              "            (self_attn): MultiheadAttention(\n",
              "              (dropout_module): FairseqDropout()\n",
              "              (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            )\n",
              "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout_module): FairseqDropout()\n",
              "            (activation_dropout_module): FairseqDropout()\n",
              "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          )\n",
              "          (9): TransformerEncoderLayer(\n",
              "            (self_attn): MultiheadAttention(\n",
              "              (dropout_module): FairseqDropout()\n",
              "              (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            )\n",
              "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout_module): FairseqDropout()\n",
              "            (activation_dropout_module): FairseqDropout()\n",
              "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          )\n",
              "          (10): TransformerEncoderLayer(\n",
              "            (self_attn): MultiheadAttention(\n",
              "              (dropout_module): FairseqDropout()\n",
              "              (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            )\n",
              "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout_module): FairseqDropout()\n",
              "            (activation_dropout_module): FairseqDropout()\n",
              "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          )\n",
              "          (11): TransformerEncoderLayer(\n",
              "            (self_attn): MultiheadAttention(\n",
              "              (dropout_module): FairseqDropout()\n",
              "              (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            )\n",
              "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout_module): FairseqDropout()\n",
              "            (activation_dropout_module): FairseqDropout()\n",
              "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          )\n",
              "          (12): TransformerEncoderLayer(\n",
              "            (self_attn): MultiheadAttention(\n",
              "              (dropout_module): FairseqDropout()\n",
              "              (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            )\n",
              "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout_module): FairseqDropout()\n",
              "            (activation_dropout_module): FairseqDropout()\n",
              "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          )\n",
              "          (13): TransformerEncoderLayer(\n",
              "            (self_attn): MultiheadAttention(\n",
              "              (dropout_module): FairseqDropout()\n",
              "              (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            )\n",
              "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout_module): FairseqDropout()\n",
              "            (activation_dropout_module): FairseqDropout()\n",
              "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          )\n",
              "          (14): TransformerEncoderLayer(\n",
              "            (self_attn): MultiheadAttention(\n",
              "              (dropout_module): FairseqDropout()\n",
              "              (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            )\n",
              "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout_module): FairseqDropout()\n",
              "            (activation_dropout_module): FairseqDropout()\n",
              "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          )\n",
              "          (15): TransformerEncoderLayer(\n",
              "            (self_attn): MultiheadAttention(\n",
              "              (dropout_module): FairseqDropout()\n",
              "              (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            )\n",
              "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout_module): FairseqDropout()\n",
              "            (activation_dropout_module): FairseqDropout()\n",
              "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          )\n",
              "          (16): TransformerEncoderLayer(\n",
              "            (self_attn): MultiheadAttention(\n",
              "              (dropout_module): FairseqDropout()\n",
              "              (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            )\n",
              "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout_module): FairseqDropout()\n",
              "            (activation_dropout_module): FairseqDropout()\n",
              "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          )\n",
              "          (17): TransformerEncoderLayer(\n",
              "            (self_attn): MultiheadAttention(\n",
              "              (dropout_module): FairseqDropout()\n",
              "              (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            )\n",
              "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout_module): FairseqDropout()\n",
              "            (activation_dropout_module): FairseqDropout()\n",
              "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          )\n",
              "          (18): TransformerEncoderLayer(\n",
              "            (self_attn): MultiheadAttention(\n",
              "              (dropout_module): FairseqDropout()\n",
              "              (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            )\n",
              "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout_module): FairseqDropout()\n",
              "            (activation_dropout_module): FairseqDropout()\n",
              "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          )\n",
              "          (19): TransformerEncoderLayer(\n",
              "            (self_attn): MultiheadAttention(\n",
              "              (dropout_module): FairseqDropout()\n",
              "              (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            )\n",
              "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout_module): FairseqDropout()\n",
              "            (activation_dropout_module): FairseqDropout()\n",
              "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          )\n",
              "          (20): TransformerEncoderLayer(\n",
              "            (self_attn): MultiheadAttention(\n",
              "              (dropout_module): FairseqDropout()\n",
              "              (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            )\n",
              "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout_module): FairseqDropout()\n",
              "            (activation_dropout_module): FairseqDropout()\n",
              "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          )\n",
              "          (21): TransformerEncoderLayer(\n",
              "            (self_attn): MultiheadAttention(\n",
              "              (dropout_module): FairseqDropout()\n",
              "              (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            )\n",
              "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout_module): FairseqDropout()\n",
              "            (activation_dropout_module): FairseqDropout()\n",
              "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          )\n",
              "          (22): TransformerEncoderLayer(\n",
              "            (self_attn): MultiheadAttention(\n",
              "              (dropout_module): FairseqDropout()\n",
              "              (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            )\n",
              "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout_module): FairseqDropout()\n",
              "            (activation_dropout_module): FairseqDropout()\n",
              "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          )\n",
              "          (23): TransformerEncoderLayer(\n",
              "            (self_attn): MultiheadAttention(\n",
              "              (dropout_module): FairseqDropout()\n",
              "              (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "            )\n",
              "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout_module): FairseqDropout()\n",
              "            (activation_dropout_module): FairseqDropout()\n",
              "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (lm_head): RobertaLMHead(\n",
              "        (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "        (layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
              "      )\n",
              "    )\n",
              "    (classification_heads): ModuleDict(\n",
              "      (task1a_head): RobertaClassificationHead(\n",
              "        (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "        (dropout): Dropout(p=0.0, inplace=False)\n",
              "        (out_proj): Linear(in_features=1024, out_features=2, bias=True)\n",
              "      )\n",
              "    )\n",
              "  )\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eAo5WZgxV9iy",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7032faa7-beea-45cf-d43e-3b6583bfd683"
      },
      "source": [
        "label_fn = lambda label: roberta.task.label_dictionary.string(\n",
        "    [label + roberta.task.label_dictionary.nspecial]\n",
        ")\n",
        "preds, labels = [], []\n",
        "for i in tqdm(range(len(X_val)), total=len(X_val)):\n",
        "  tokens = roberta.encode(X_val[i])\n",
        "  pred = label_fn(roberta.predict('task1a_head',tokens).argmax().item())\n",
        "  preds.append(pred)\n",
        "  labels.append(Y_val[i])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 1826/1826 [15:33<00:00,  1.96it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 197
        },
        "id": "GkBqqPl3jAnu",
        "outputId": "6b3223a7-8d15-4910-8246-2a29ea4ec0b2"
      },
      "source": [
        "import pandas as pd\n",
        "from sklearn.metrics import classification_report\n",
        "df_preds=pd.read_csv(\"/content/val_final.tsv\", sep='\\t')\n",
        "df_label=pd.read_csv(\"/content/class.tsv\", sep='\\t')\n",
        "df=df_preds.merge(df_label, on=\"tweet_id\")\n",
        "df.columns=[\"tweet_id\",\"preds\",\"label\"]\n",
        "df['label'].replace({\"NoADE\":0, \"ADE\":1}, inplace=True)\n",
        "df['preds'].replace({\"NoADE\":0, \"ADE\":1}, inplace=True)\n",
        "df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tweet_id</th>\n",
              "      <th>preds</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>342138828087234560</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>342382016030973953</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>345414697148887041</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>348207562631557122</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>326596903791902720</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "             tweet_id  preds  label\n",
              "0  342138828087234560      1      1\n",
              "1  342382016030973953      1      1\n",
              "2  345414697148887041      1      1\n",
              "3  348207562631557122      1      1\n",
              "4  326596903791902720      1      1"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jAKKY9xoXuxX",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "eefc4e2b-dc81-499a-a68e-237affe04779"
      },
      "source": [
        "import pandas as pd\n",
        "from sklearn.metrics import classification_report\n",
        "preds=df[\"preds\"]\n",
        "labels=df[\"label\"]\n",
        "report = classification_report(labels, list(map(int, preds)))\n",
        "print(report)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.99      0.95      0.97       848\n",
            "           1       0.59      0.92      0.72        65\n",
            "\n",
            "    accuracy                           0.95       913\n",
            "   macro avg       0.79      0.94      0.85       913\n",
            "weighted avg       0.97      0.95      0.95       913\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hgoNQaLx4hgy"
      },
      "source": [
        "!rm checkpoints/checkpoint1.pt checkpoints/checkpoint2.pt checkpoints/checkpoint3.pt checkpoints/checkpoint4.pt"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-y2iYfjDXe4k"
      },
      "source": [
        "# Running on the Validation Set\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mjr3e94OnGPr"
      },
      "source": [
        "df_tweets = pd.read_csv('/content/tweets.tsv', sep = '\\t')\n",
        "df_class = pd.read_csv('/content/class.tsv', sep = '\\t')\n",
        "df_valid = pd.merge(df_tweets, df_class, on = 'tweet_id')\n",
        "df_valid['label'].replace({\"NoADE\":0, \"ADE\":1}, inplace=True)\n",
        "df_valid.head()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "74-PDsjidSqa"
      },
      "source": [
        "df_test = pd.read_csv('/content/drive/MyDrive/UPENN/test_tweets.tsv', sep='\\t')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ObvVUy4JwGWv",
        "outputId": "3ed6746b-3c7b-4b77-9052-8755375bdb9f"
      },
      "source": [
        "index_names = df_test[df_test['tweet'].str.split().apply(len)>35].index\n",
        "df_test.drop(index_names, inplace=True)\n",
        "df_test.tweet_id.nunique()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "10472"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YxeJBmHXV0Ni",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4013168f-e2e1-49ee-f2af-646898cdc746"
      },
      "source": [
        "label_fn = lambda label: roberta.task.label_dictionary.string(\n",
        "    [label + roberta.task.label_dictionary.nspecial]\n",
        ")\n",
        "preds, id = [], []\n",
        "for index, row in tqdm(df_test.iterrows(), total=len(df_test)):\n",
        "  tokens = roberta.encode(row[\"tweet\"])\n",
        "  pred = label_fn(roberta.predict('task1a_head',tokens).argmax().item())\n",
        "  preds.append(pred)\n",
        "  id.append(row[\"tweet_id\"])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 10472/10472 [1:26:06<00:00,  2.03it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "2LUwJ7h4hRq_",
        "outputId": "c9ef1590-2c2f-49ac-92f1-59bf92a4108c"
      },
      "source": [
        "df_1a = pd.DataFrame(list(zip(id, preds)), columns = ['tweet_id', 'label'])\n",
        "df_1a['label']=df_1a['label'].replace({0:\"NoADE\", 1:\"ADE\"})\n",
        "df_1a.reset_index(drop=True, inplace=True)\n",
        "df_1a.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tweet_id</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>924202174481162243</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1065029162904182785</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>487283084916572161</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>537586315932729344</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>530198807435149312</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "              tweet_id label\n",
              "0   924202174481162243     0\n",
              "1  1065029162904182785     0\n",
              "2   487283084916572161     0\n",
              "3   537586315932729344     1\n",
              "4   530198807435149312     0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WavrQTHmjKrb"
      },
      "source": [
        "df_1a.to_csv(\"/content/drive/MyDrive/UPENN/1a_sub2.tsv\", sep='\\t')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iYkLS_ybXY_w"
      },
      "source": [
        "from sklearn.metrics import classification_report\n",
        "report = classification_report(labels, list(map(int, preds)))\n",
        "print(report)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LC5osXMnNiUy",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "923045e8-fb7b-455c-a24e-5f6c5e5bb7e7"
      },
      "source": [
        "print(Counter(preds))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Counter({'0': 9543, '1': 929})\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3buJ8nzudJqA",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "outputId": "c5a6cd29-b8de-4cb9-a26b-8f356b7d7b44"
      },
      "source": [
        "df_preds = pd.DataFrame(preds, columns = ['Predictions'])\n",
        "df_id = pd.DataFrame(df_valid['tweet_id'], columns = ['tweet_id'])\n",
        "df_results = pd.concat([df_id, df_preds], join = 'outer', axis = 1)\n",
        "df_results.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tweet_id</th>\n",
              "      <th>Predictions</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>3.421388e+17</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>3.423820e+17</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3.454147e+17</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3.482076e+17</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>3.265969e+17</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "       tweet_id Predictions\n",
              "0  3.421388e+17           0\n",
              "1  3.423820e+17           0\n",
              "2  3.454147e+17           0\n",
              "3  3.482076e+17           1\n",
              "4  3.265969e+17           0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I-YANP6PN0TU"
      },
      "source": [
        "df_results.to_csv('/content/val.tsv', sep = '\\t')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZKAUNcx4NT98"
      },
      "source": [
        "len(df_id)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CQzSztaSN9ro",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "outputId": "eff11117-434b-425b-a91f-d52b6cb4a0f9"
      },
      "source": [
        "import pandas as pd\n",
        "df = pd.read_csv('/content/drive/MyDrive/UPENN/1a_sub2.tsv', sep = '\\t')\n",
        "df.drop([\"Unnamed: 0\"], axis=1, inplace=True)\n",
        "df.columns = [\"tweet_id\", \"label\"]\n",
        "df['label'].replace({0:\"NoADE\", 1:\"ADE\"}, inplace=True)\n",
        "df.reset_index(drop=True, inplace=True)\n",
        "df = df[df.label==\"ADE\"]\n",
        "df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tweet_id</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>537586315932729344</td>\n",
              "      <td>ADE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26</th>\n",
              "      <td>430403326899798017</td>\n",
              "      <td>ADE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27</th>\n",
              "      <td>422143636730441728</td>\n",
              "      <td>ADE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35</th>\n",
              "      <td>532252559600517121</td>\n",
              "      <td>ADE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>36</th>\n",
              "      <td>449547966148542464</td>\n",
              "      <td>ADE</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "              tweet_id label\n",
              "3   537586315932729344   ADE\n",
              "26  430403326899798017   ADE\n",
              "27  422143636730441728   ADE\n",
              "35  532252559600517121   ADE\n",
              "36  449547966148542464   ADE"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ji-DHPB_F6i9"
      },
      "source": [
        "df.to_csv('/content/test_sub2.tsv', sep = '\\t', index= False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H0BlVz1MHbEJ",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "outputId": "17540b48-c3f5-43c8-a159-9fd2220fb6bf"
      },
      "source": [
        "import pandas as pd\n",
        "df_1a = pd.read_csv('/content/test_sub2.tsv', sep = '\\t') \n",
        "df_1b = pd.read_csv('/content/1b_new.tsv', sep = '\\t')\n",
        "df_1b.drop([\"Unnamed: 0\"], axis=1, inplace=True)\n",
        "df_1b.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tweet_id</th>\n",
              "      <th>start</th>\n",
              "      <th>end</th>\n",
              "      <th>span</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>487283084916572161</td>\n",
              "      <td>28</td>\n",
              "      <td>40</td>\n",
              "      <td>stress nigga</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>537586315932729344</td>\n",
              "      <td>51</td>\n",
              "      <td>62</td>\n",
              "      <td>mouth taste</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>529265954023174144</td>\n",
              "      <td>87</td>\n",
              "      <td>98</td>\n",
              "      <td>defenseless</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>529265954023174144</td>\n",
              "      <td>132</td>\n",
              "      <td>136</td>\n",
              "      <td>moth</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>503250570736844800</td>\n",
              "      <td>53</td>\n",
              "      <td>63</td>\n",
              "      <td>no worries</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "             tweet_id  start  end          span\n",
              "0  487283084916572161     28   40  stress nigga\n",
              "1  537586315932729344     51   62   mouth taste\n",
              "2  529265954023174144     87   98   defenseless\n",
              "3  529265954023174144    132  136          moth\n",
              "4  503250570736844800     53   63    no worries"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "jI64MhVHFnWG",
        "outputId": "c2f93d1d-2ef4-4e01-f340-43a79e6ed88e"
      },
      "source": [
        "df_1 = df_1a.merge(df_1b, on = 'tweet_id')\n",
        "df_1.columns = [\"tweet_id\", \"label\", \"start\", \"end\", \"span\"]\n",
        "df_1[\"start\"] = df_1[\"start\"].astype(int)\n",
        "df_1[\"end\"] = df_1[\"end\"].astype(int)\n",
        "df_1.dropna(axis=0, inplace=True)\n",
        "df_1 = df_1[df_1.label==\"ADE\"]\n",
        "df_1.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tweet_id</th>\n",
              "      <th>label</th>\n",
              "      <th>start</th>\n",
              "      <th>end</th>\n",
              "      <th>span</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>537586315932729344</td>\n",
              "      <td>ADE</td>\n",
              "      <td>51</td>\n",
              "      <td>62</td>\n",
              "      <td>mouth taste</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>430403326899798017</td>\n",
              "      <td>ADE</td>\n",
              "      <td>69</td>\n",
              "      <td>79</td>\n",
              "      <td>withdrawal</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>422143636730441728</td>\n",
              "      <td>ADE</td>\n",
              "      <td>3</td>\n",
              "      <td>27</td>\n",
              "      <td>memory is still so awful</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>532252559600517121</td>\n",
              "      <td>ADE</td>\n",
              "      <td>0</td>\n",
              "      <td>18</td>\n",
              "      <td>got off the heroin</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>532252559600517121</td>\n",
              "      <td>ADE</td>\n",
              "      <td>52</td>\n",
              "      <td>61</td>\n",
              "      <td>addiction</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "             tweet_id label  start  end                      span\n",
              "0  537586315932729344   ADE     51   62               mouth taste\n",
              "1  430403326899798017   ADE     69   79                withdrawal\n",
              "2  422143636730441728   ADE      3   27  memory is still so awful\n",
              "3  532252559600517121   ADE      0   18        got off the heroin\n",
              "4  532252559600517121   ADE     52   61                 addiction"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w66AWd664kb8"
      },
      "source": [
        "df_1.to_csv('/content/testb_final.tsv', sep = '\\t', index= False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OyJSan7EP_H9"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}