NER model file.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"accelerator":"GPU","colab":{"name":"NER project1.ipynb","provenance":[],"collapsed_sections":[],"toc_visible":true,"machine_shape":"hm"},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"code","metadata":{"id":"rGrH5t-WlbSg"},"source":["#Named Entity Recognition using GMB(Groningen Meaning Bank) dataset"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"fFR96W-p3dsh","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1619956424988,"user_tz":-300,"elapsed":874,"user":{"displayName":"AI Team","photoUrl":"","userId":"17689037188246991143"}},"outputId":"6f2bd44b-88b7-4739-dbb6-b5b623f48a83"},"source":["from google.colab import drive\n","drive.mount(\"/content/gdrive\")"],"execution_count":9,"outputs":[{"output_type":"stream","text":["Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount(\"/content/gdrive\", force_remount=True).\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"kp01YSQn3J87"},"source":["## Download data from kagle"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"DVlclL9PlekW","executionInfo":{"elapsed":9896,"status":"ok","timestamp":1619955958387,"user":{"displayName":"AI Team","photoUrl":"","userId":"17689037188246991143"},"user_tz":-300},"outputId":"043e24c5-e173-4f7f-8e18-c4d7f7f3b7b2"},"source":["!pip install kaggle\n","!pip install pyunpack\n","!pip install patool"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Requirement already satisfied: kaggle in /usr/local/lib/python3.7/dist-packages (1.5.12)\n","Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from kaggle) (4.41.1)\n","Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from kaggle) (2.23.0)\n","Requirement already satisfied: certifi in /usr/local/lib/python3.7/dist-packages (from kaggle) (2020.12.5)\n","Requirement already satisfied: python-dateutil in /usr/local/lib/python3.7/dist-packages (from kaggle) (2.8.1)\n","Requirement already satisfied: six>=1.10 in /usr/local/lib/python3.7/dist-packages (from kaggle) (1.15.0)\n","Requirement already satisfied: python-slugify in /usr/local/lib/python3.7/dist-packages (from kaggle) (4.0.1)\n","Requirement already satisfied: urllib3 in /usr/local/lib/python3.7/dist-packages (from kaggle) (1.24.3)\n","Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->kaggle) (3.0.4)\n","Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->kaggle) (2.10)\n","Requirement already satisfied: text-unidecode>=1.3 in /usr/local/lib/python3.7/dist-packages (from python-slugify->kaggle) (1.3)\n","Collecting pyunpack\n","  Downloading https://files.pythonhosted.org/packages/83/29/020436b1d8e96e5f26fa282b9c3c13a3b456a36b9ea2edc87c5fed008369/pyunpack-0.2.2-py2.py3-none-any.whl\n","Collecting easyprocess\n","  Downloading https://files.pythonhosted.org/packages/48/3c/75573613641c90c6d094059ac28adb748560d99bd27ee6f80cce398f404e/EasyProcess-0.3-py2.py3-none-any.whl\n","Collecting entrypoint2\n","  Downloading https://files.pythonhosted.org/packages/8a/b0/8ef4b1d8be02448d164c52466530059d7f57218655d21309a0c4236d7454/entrypoint2-0.2.4-py3-none-any.whl\n","Installing collected packages: easyprocess, entrypoint2, pyunpack\n","Successfully installed easyprocess-0.3 entrypoint2-0.2.4 pyunpack-0.2.2\n","Collecting patool\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/43/94/52243ddff508780dd2d8110964320ab4851134a55ab102285b46e740f76a/patool-1.12-py2.py3-none-any.whl (77kB)\n","\u001b[K     |████████████████████████████████| 81kB 4.0MB/s \n","\u001b[?25hInstalling collected packages: patool\n","Successfully installed patool-1.12\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"zQVH1G-SlehL"},"source":["#upload the credentials\n","from google.colab import files\n","files.upload()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"1Al297oyled8","executionInfo":{"elapsed":3248,"status":"ok","timestamp":1619770989070,"user":{"displayName":"aftab anjum","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gjr9EAQIKikAsQFruoHrEH12B72Qi-qbRgbTxCSNg=s64","userId":"09823110291100232267"},"user_tz":-300},"outputId":"c4de9d0b-1f4d-40f8-fd62-30d0ed58bb39"},"source":["#The Kaggle API client expects the json file to be in ~/.kaggle folder so let’s create a new folder and move it inside.\n","!mkdir -p ~/.kaggle\n","!cp kaggle.json ~/.kaggle/\n","!chmod 600 ~/.kaggle/kaggle.json\n","!pip install --upgrade --force-reinstall --no-deps kaggle"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Collecting kaggle\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/3a/e7/3bac01547d2ed3d308ac92a0878fbdb0ed0f3d41fb1906c319ccbba1bfbc/kaggle-1.5.12.tar.gz (58kB)\n","\r\u001b[K     |█████▋                          | 10kB 21.1MB/s eta 0:00:01\r\u001b[K     |███████████▏                    | 20kB 26.3MB/s eta 0:00:01\r\u001b[K     |████████████████▊               | 30kB 32.1MB/s eta 0:00:01\r\u001b[K     |██████████████████████▎         | 40kB 20.1MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▉    | 51kB 12.8MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 61kB 3.0MB/s \n","\u001b[?25hBuilding wheels for collected packages: kaggle\n","  Building wheel for kaggle (setup.py) ... \u001b[?25l\u001b[?25hdone\n","  Created wheel for kaggle: filename=kaggle-1.5.12-cp37-none-any.whl size=73053 sha256=71c10421f520109723da55c9dcb2ea91a306638af5f20436b9f1c1054541f518\n","  Stored in directory: /root/.cache/pip/wheels/a1/6a/26/d30b7499ff85a4a4593377a87ecf55f7d08af42f0de9b60303\n","Successfully built kaggle\n","Installing collected packages: kaggle\n","  Found existing installation: kaggle 1.5.12\n","    Uninstalling kaggle-1.5.12:\n","      Successfully uninstalled kaggle-1.5.12\n","Successfully installed kaggle-1.5.12\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"jcwYLeXelean","executionInfo":{"elapsed":7288,"status":"ok","timestamp":1619771044211,"user":{"displayName":"aftab anjum","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gjr9EAQIKikAsQFruoHrEH12B72Qi-qbRgbTxCSNg=s64","userId":"09823110291100232267"},"user_tz":-300},"outputId":"8d8da83a-63e9-44e5-f80a-83263aed80d0"},"source":["!kaggle datasets download -d abhinavwalia95/entity-annotated-corpus\n","#Let’s see the imported files:\n","!ls"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Downloading entity-annotated-corpus.zip to /content\n"," 45% 12.0M/26.4M [00:02<00:04, 3.57MB/s]\n","100% 26.4M/26.4M [00:03<00:00, 9.16MB/s]\n","entity-annotated-corpus.zip  gdrive  kaggle.json  sample_data\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"Nje2X2sZleW_"},"source":["#put file name and the dic to store file \n","from pyunpack import Archive\n","Archive('entity-annotated-corpus.zip').extractall('/content/gdrive/MyDrive/NLP projects/NER projects/project 1')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":204},"id":"kMOLYe4U3YTQ","executionInfo":{"elapsed":2382,"status":"ok","timestamp":1619861292793,"user":{"displayName":"aftab anjum","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gjr9EAQIKikAsQFruoHrEH12B72Qi-qbRgbTxCSNg=s64","userId":"09823110291100232267"},"user_tz":-300},"outputId":"3f11ac85-d95b-460d-c148-5733f285ed3d"},"source":["import pandas as pd\n","import pandas as pd\n","df_NER_train = pd.read_csv(\"/content/gdrive/MyDrive/NLP projects/NER projects/project 1/ner_dataset.csv\", encoding=\"latin1\")\n","df_NER_train.head()\n"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>Sentence #</th>\n","      <th>Word</th>\n","      <th>POS</th>\n","      <th>Tag</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>Sentence: 1</td>\n","      <td>Thousands</td>\n","      <td>NNS</td>\n","      <td>O</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>NaN</td>\n","      <td>of</td>\n","      <td>IN</td>\n","      <td>O</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>NaN</td>\n","      <td>demonstrators</td>\n","      <td>NNS</td>\n","      <td>O</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>NaN</td>\n","      <td>have</td>\n","      <td>VBP</td>\n","      <td>O</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>NaN</td>\n","      <td>marched</td>\n","      <td>VBN</td>\n","      <td>O</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["    Sentence #           Word  POS Tag\n","0  Sentence: 1      Thousands  NNS   O\n","1          NaN             of   IN   O\n","2          NaN  demonstrators  NNS   O\n","3          NaN           have  VBP   O\n","4          NaN        marched  VBN   O"]},"metadata":{"tags":[]},"execution_count":2}]},{"cell_type":"code","metadata":{"id":"T3wCEK2VsV7w"},"source":["import numpy as np\n","train, validate, test = \\\n","              np.split(df_NER_train.sample(frac=1, random_state=42), \n","                       [int(.85*len(df_NER_train)), int(.925*len(df_NER_train))])"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":204},"id":"PBsMadmysn4z","executionInfo":{"elapsed":752,"status":"ok","timestamp":1619861298224,"user":{"displayName":"aftab anjum","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gjr9EAQIKikAsQFruoHrEH12B72Qi-qbRgbTxCSNg=s64","userId":"09823110291100232267"},"user_tz":-300},"outputId":"c5bae7aa-4dd7-4004-9874-53df5e06ee75"},"source":["train.head()"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>Sentence #</th>\n","      <th>Word</th>\n","      <th>POS</th>\n","      <th>Tag</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>781974</th>\n","      <td>NaN</td>\n","      <td>the</td>\n","      <td>DT</td>\n","      <td>O</td>\n","    </tr>\n","    <tr>\n","      <th>937737</th>\n","      <td>NaN</td>\n","      <td>investors</td>\n","      <td>NNS</td>\n","      <td>O</td>\n","    </tr>\n","    <tr>\n","      <th>907828</th>\n","      <td>NaN</td>\n","      <td>New</td>\n","      <td>NNP</td>\n","      <td>B-org</td>\n","    </tr>\n","    <tr>\n","      <th>784628</th>\n","      <td>NaN</td>\n","      <td>his</td>\n","      <td>PRP$</td>\n","      <td>O</td>\n","    </tr>\n","    <tr>\n","      <th>662460</th>\n","      <td>NaN</td>\n","      <td>the</td>\n","      <td>DT</td>\n","      <td>O</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["       Sentence #       Word   POS    Tag\n","781974        NaN        the    DT      O\n","937737        NaN  investors   NNS      O\n","907828        NaN        New   NNP  B-org\n","784628        NaN        his  PRP$      O\n","662460        NaN        the    DT      O"]},"metadata":{"tags":[]},"execution_count":4}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"SzowIOlXsyY1","executionInfo":{"elapsed":2081,"status":"ok","timestamp":1619855485822,"user":{"displayName":"aftab anjum","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gjr9EAQIKikAsQFruoHrEH12B72Qi-qbRgbTxCSNg=s64","userId":"09823110291100232267"},"user_tz":-300},"outputId":"546e9656-6f5e-47f8-e134-d2b3588dab7b"},"source":["train.shape,test.shape,validate.shape"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["((891288, 4), (78644, 4), (78643, 4))"]},"metadata":{"tags":[]},"execution_count":5}]},{"cell_type":"code","metadata":{"id":"ASen8TzBvDS5"},"source":["train= train.reset_index(drop=True)\n","test= test.reset_index(drop=True)\n","validate= validate.reset_index(drop=True)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Exu5eCaW3YPw"},"source":["def flair_datformate(df_NER):\n","    df_NER=df_NER\n","    my_dic = {'docs':[], 'tags':[]}\n","    for i in range(len(df_NER)):\n","      if df_NER['Word'][i]!='.':\n","        my_dic['docs'].append(df_NER['Word'][i])\n","        my_dic['tags'].append(df_NER['Tag'][i])\n","      else:\n","        my_dic['docs'].append('.')\n","        my_dic['tags'].append('.')\n","        my_dic['docs'].append(' ')\n","        my_dic['tags'].append(' ')\n","    return my_dic"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"fYUcCYgsmsFv"},"source":["train_dic = flair_datformate(train)\n","valid_dic = flair_datformate(test)\n","test_dic = flair_datformate(validate)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":1000},"id":"oSO87iA9msCH","executionInfo":{"elapsed":11300,"status":"ok","timestamp":1619855507661,"user":{"displayName":"aftab anjum","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gjr9EAQIKikAsQFruoHrEH12B72Qi-qbRgbTxCSNg=s64","userId":"09823110291100232267"},"user_tz":-300},"outputId":"fa36e130-0477-4706-daa3-9d246615d099"},"source":["final_train = pd.DataFrame.from_dict(train_dic)\n","final_valid = pd.DataFrame.from_dict(valid_dic)\n","final_test = pd.DataFrame.from_dict(test_dic)\n","final_train[:50]"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>docs</th>\n","      <th>tags</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>the</td>\n","      <td>O</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>investors</td>\n","      <td>O</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>New</td>\n","      <td>B-org</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>his</td>\n","      <td>O</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>the</td>\n","      <td>O</td>\n","    </tr>\n","    <tr>\n","      <th>5</th>\n","      <td>Earth</td>\n","      <td>B-geo</td>\n","    </tr>\n","    <tr>\n","      <th>6</th>\n","      <td>modest</td>\n","      <td>O</td>\n","    </tr>\n","    <tr>\n","      <th>7</th>\n","      <td>on</td>\n","      <td>O</td>\n","    </tr>\n","    <tr>\n","      <th>8</th>\n","      <td>war</td>\n","      <td>O</td>\n","    </tr>\n","    <tr>\n","      <th>9</th>\n","      <td>,</td>\n","      <td>O</td>\n","    </tr>\n","    <tr>\n","      <th>10</th>\n","      <td>officers</td>\n","      <td>O</td>\n","    </tr>\n","    <tr>\n","      <th>11</th>\n","      <td>prompted</td>\n","      <td>O</td>\n","    </tr>\n","    <tr>\n","      <th>12</th>\n","      <td>Sri</td>\n","      <td>B-per</td>\n","    </tr>\n","    <tr>\n","      <th>13</th>\n","      <td>monthly</td>\n","      <td>O</td>\n","    </tr>\n","    <tr>\n","      <th>14</th>\n","      <td>political</td>\n","      <td>O</td>\n","    </tr>\n","    <tr>\n","      <th>15</th>\n","      <td>former</td>\n","      <td>O</td>\n","    </tr>\n","    <tr>\n","      <th>16</th>\n","      <td>Palestinian</td>\n","      <td>B-org</td>\n","    </tr>\n","    <tr>\n","      <th>17</th>\n","      <td>eliminate</td>\n","      <td>O</td>\n","    </tr>\n","    <tr>\n","      <th>18</th>\n","      <td>60</td>\n","      <td>O</td>\n","    </tr>\n","    <tr>\n","      <th>19</th>\n","      <td>.</td>\n","      <td>.</td>\n","    </tr>\n","    <tr>\n","      <th>20</th>\n","      <td></td>\n","      <td></td>\n","    </tr>\n","    <tr>\n","      <th>21</th>\n","      <td>Haiti</td>\n","      <td>B-geo</td>\n","    </tr>\n","    <tr>\n","      <th>22</th>\n","      <td>Jordan</td>\n","      <td>B-gpe</td>\n","    </tr>\n","    <tr>\n","      <th>23</th>\n","      <td>under</td>\n","      <td>O</td>\n","    </tr>\n","    <tr>\n","      <th>24</th>\n","      <td>Hezbollah</td>\n","      <td>B-org</td>\n","    </tr>\n","    <tr>\n","      <th>25</th>\n","      <td>dialogue</td>\n","      <td>O</td>\n","    </tr>\n","    <tr>\n","      <th>26</th>\n","      <td>Harry</td>\n","      <td>B-per</td>\n","    </tr>\n","    <tr>\n","      <th>27</th>\n","      <td>was</td>\n","      <td>O</td>\n","    </tr>\n","    <tr>\n","      <th>28</th>\n","      <td>assassinated</td>\n","      <td>O</td>\n","    </tr>\n","    <tr>\n","      <th>29</th>\n","      <td>an</td>\n","      <td>O</td>\n","    </tr>\n","    <tr>\n","      <th>30</th>\n","      <td>President</td>\n","      <td>B-per</td>\n","    </tr>\n","    <tr>\n","      <th>31</th>\n","      <td>been</td>\n","      <td>O</td>\n","    </tr>\n","    <tr>\n","      <th>32</th>\n","      <td>was</td>\n","      <td>O</td>\n","    </tr>\n","    <tr>\n","      <th>33</th>\n","      <td>at</td>\n","      <td>O</td>\n","    </tr>\n","    <tr>\n","      <th>34</th>\n","      <td>votes</td>\n","      <td>O</td>\n","    </tr>\n","    <tr>\n","      <th>35</th>\n","      <td>rocket</td>\n","      <td>O</td>\n","    </tr>\n","    <tr>\n","      <th>36</th>\n","      <td>Chinese</td>\n","      <td>B-gpe</td>\n","    </tr>\n","    <tr>\n","      <th>37</th>\n","      <td>economy</td>\n","      <td>O</td>\n","    </tr>\n","    <tr>\n","      <th>38</th>\n","      <td>.</td>\n","      <td>.</td>\n","    </tr>\n","    <tr>\n","      <th>39</th>\n","      <td></td>\n","      <td></td>\n","    </tr>\n","    <tr>\n","      <th>40</th>\n","      <td>soared</td>\n","      <td>O</td>\n","    </tr>\n","    <tr>\n","      <th>41</th>\n","      <td>with</td>\n","      <td>O</td>\n","    </tr>\n","    <tr>\n","      <th>42</th>\n","      <td>of</td>\n","      <td>O</td>\n","    </tr>\n","    <tr>\n","      <th>43</th>\n","      <td>going</td>\n","      <td>O</td>\n","    </tr>\n","    <tr>\n","      <th>44</th>\n","      <td>talks</td>\n","      <td>O</td>\n","    </tr>\n","    <tr>\n","      <th>45</th>\n","      <td>condone</td>\n","      <td>O</td>\n","    </tr>\n","    <tr>\n","      <th>46</th>\n","      <td>the</td>\n","      <td>O</td>\n","    </tr>\n","    <tr>\n","      <th>47</th>\n","      <td>In</td>\n","      <td>O</td>\n","    </tr>\n","    <tr>\n","      <th>48</th>\n","      <td>responsible</td>\n","      <td>O</td>\n","    </tr>\n","    <tr>\n","      <th>49</th>\n","      <td>Palestinian</td>\n","      <td>B-gpe</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["            docs   tags\n","0            the      O\n","1      investors      O\n","2            New  B-org\n","3            his      O\n","4            the      O\n","5          Earth  B-geo\n","6         modest      O\n","7             on      O\n","8            war      O\n","9              ,      O\n","10      officers      O\n","11      prompted      O\n","12           Sri  B-per\n","13       monthly      O\n","14     political      O\n","15        former      O\n","16   Palestinian  B-org\n","17     eliminate      O\n","18            60      O\n","19             .      .\n","20                     \n","21         Haiti  B-geo\n","22        Jordan  B-gpe\n","23         under      O\n","24     Hezbollah  B-org\n","25      dialogue      O\n","26         Harry  B-per\n","27           was      O\n","28  assassinated      O\n","29            an      O\n","30     President  B-per\n","31          been      O\n","32           was      O\n","33            at      O\n","34         votes      O\n","35        rocket      O\n","36       Chinese  B-gpe\n","37       economy      O\n","38             .      .\n","39                     \n","40        soared      O\n","41          with      O\n","42            of      O\n","43         going      O\n","44         talks      O\n","45       condone      O\n","46           the      O\n","47            In      O\n","48   responsible      O\n","49   Palestinian  B-gpe"]},"metadata":{"tags":[]},"execution_count":10}]},{"cell_type":"code","metadata":{"id":"t0V9vTnOmr-k"},"source":["np.savetxt('/content/gdrive/MyDrive/NLP projects/NER projects/project 1/train.txt', final_train.values, fmt='%s', delimiter=\" \", header=\"-DOCSTART- O\") \n","np.savetxt('/content/gdrive/MyDrive/NLP projects/NER projects/project 1/testa.txt', final_valid.values, fmt='%s', delimiter=\" \", header=\"-DOCSTART- O\") \n","np.savetxt('/content/gdrive/MyDrive/NLP projects/NER projects/project 1/testb.txt', final_test.values, fmt='%s', delimiter=\" \", header=\"-DOCSTART- O\") "],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"UVAViAa1mr6z"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"8yLgisOkxkwd"},"source":["## Flair model"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"t2xI8uy23YMf","executionInfo":{"status":"ok","timestamp":1619956434472,"user_tz":-300,"elapsed":3267,"user":{"displayName":"AI Team","photoUrl":"","userId":"17689037188246991143"}},"outputId":"a620c8d7-efc8-438c-9ebc-eea6fde19a35"},"source":["!pip install flair"],"execution_count":10,"outputs":[{"output_type":"stream","text":["Requirement already satisfied: flair in /usr/local/lib/python3.7/dist-packages (0.8.0.post1)\n","Requirement already satisfied: hyperopt>=0.1.1 in /usr/local/lib/python3.7/dist-packages (from flair) (0.1.2)\n","Requirement already satisfied: matplotlib>=2.2.3 in /usr/local/lib/python3.7/dist-packages (from flair) (3.2.2)\n","Requirement already satisfied: tabulate in /usr/local/lib/python3.7/dist-packages (from flair) (0.8.9)\n","Requirement already satisfied: gensim<=3.8.3,>=3.4.0 in /usr/local/lib/python3.7/dist-packages (from flair) (3.6.0)\n","Requirement already satisfied: bpemb>=0.3.2 in /usr/local/lib/python3.7/dist-packages (from flair) (0.3.3)\n","Requirement already satisfied: tqdm>=4.26.0 in /usr/local/lib/python3.7/dist-packages (from flair) (4.41.1)\n","Requirement already satisfied: langdetect in /usr/local/lib/python3.7/dist-packages (from flair) (1.0.8)\n","Requirement already satisfied: torch<=1.7.1,>=1.5.0 in /usr/local/lib/python3.7/dist-packages (from flair) (1.7.1)\n","Requirement already satisfied: python-dateutil>=2.6.1 in /usr/local/lib/python3.7/dist-packages (from flair) (2.8.1)\n","Requirement already satisfied: scikit-learn>=0.21.3 in /usr/local/lib/python3.7/dist-packages (from flair) (0.22.2.post1)\n","Requirement already satisfied: deprecated>=1.2.4 in /usr/local/lib/python3.7/dist-packages (from flair) (1.2.12)\n","Requirement already satisfied: transformers>=4.0.0 in /usr/local/lib/python3.7/dist-packages (from flair) (4.5.1)\n","Requirement already satisfied: regex in /usr/local/lib/python3.7/dist-packages (from flair) (2019.12.20)\n","Requirement already satisfied: konoha<5.0.0,>=4.0.0 in /usr/local/lib/python3.7/dist-packages (from flair) (4.6.4)\n","Requirement already satisfied: huggingface-hub in /usr/local/lib/python3.7/dist-packages (from flair) (0.0.8)\n","Requirement already satisfied: lxml in /usr/local/lib/python3.7/dist-packages (from flair) (4.2.6)\n","Requirement already satisfied: ftfy in /usr/local/lib/python3.7/dist-packages (from flair) (6.0.1)\n","Requirement already satisfied: janome in /usr/local/lib/python3.7/dist-packages (from flair) (0.4.1)\n","Requirement already satisfied: sqlitedict>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from flair) (1.7.0)\n","Requirement already satisfied: sentencepiece==0.1.95 in /usr/local/lib/python3.7/dist-packages (from flair) (0.1.95)\n","Requirement already satisfied: numpy<1.20.0 in /usr/local/lib/python3.7/dist-packages (from flair) (1.19.5)\n","Requirement already satisfied: segtok>=1.5.7 in /usr/local/lib/python3.7/dist-packages (from flair) (1.5.10)\n","Requirement already satisfied: mpld3==0.3 in /usr/local/lib/python3.7/dist-packages (from flair) (0.3)\n","Requirement already satisfied: gdown==3.12.2 in /usr/local/lib/python3.7/dist-packages (from flair) (3.12.2)\n","Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from hyperopt>=0.1.1->flair) (1.15.0)\n","Requirement already satisfied: pymongo in /usr/local/lib/python3.7/dist-packages (from hyperopt>=0.1.1->flair) (3.11.3)\n","Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from hyperopt>=0.1.1->flair) (1.4.1)\n","Requirement already satisfied: future in /usr/local/lib/python3.7/dist-packages (from hyperopt>=0.1.1->flair) (0.16.0)\n","Requirement already satisfied: networkx in /usr/local/lib/python3.7/dist-packages (from hyperopt>=0.1.1->flair) (2.5.1)\n","Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=2.2.3->flair) (2.4.7)\n","Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=2.2.3->flair) (1.3.1)\n","Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=2.2.3->flair) (0.10.0)\n","Requirement already satisfied: smart-open>=1.2.1 in /usr/local/lib/python3.7/dist-packages (from gensim<=3.8.3,>=3.4.0->flair) (5.0.0)\n","Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from bpemb>=0.3.2->flair) (2.23.0)\n","Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch<=1.7.1,>=1.5.0->flair) (3.7.4.3)\n","Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn>=0.21.3->flair) (1.0.1)\n","Requirement already satisfied: wrapt<2,>=1.10 in /usr/local/lib/python3.7/dist-packages (from deprecated>=1.2.4->flair) (1.12.1)\n","Requirement already satisfied: sacremoses in /usr/local/lib/python3.7/dist-packages (from transformers>=4.0.0->flair) (0.0.45)\n","Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers>=4.0.0->flair) (3.0.12)\n","Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers>=4.0.0->flair) (3.10.1)\n","Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers>=4.0.0->flair) (20.9)\n","Requirement already satisfied: tokenizers<0.11,>=0.10.1 in /usr/local/lib/python3.7/dist-packages (from transformers>=4.0.0->flair) (0.10.2)\n","Requirement already satisfied: overrides<4.0.0,>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from konoha<5.0.0,>=4.0.0->flair) (3.1.0)\n","Requirement already satisfied: wcwidth in /usr/local/lib/python3.7/dist-packages (from ftfy->flair) (0.2.5)\n","Requirement already satisfied: decorator<5,>=4.3 in /usr/local/lib/python3.7/dist-packages (from networkx->hyperopt>=0.1.1->flair) (4.4.2)\n","Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->bpemb>=0.3.2->flair) (1.24.3)\n","Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->bpemb>=0.3.2->flair) (2.10)\n","Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->bpemb>=0.3.2->flair) (3.0.4)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->bpemb>=0.3.2->flair) (2020.12.5)\n","Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers>=4.0.0->flair) (7.1.2)\n","Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers>=4.0.0->flair) (3.4.1)\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"4T2n8xk03YJB","executionInfo":{"status":"ok","timestamp":1619956468047,"user_tz":-300,"elapsed":18043,"user":{"displayName":"AI Team","photoUrl":"","userId":"17689037188246991143"}},"outputId":"b3485232-ebca-41cb-8b4e-85a591676de1"},"source":["from flair.data import Corpus\n","from flair.datasets import TREC_6\n","from flair.embeddings import WordEmbeddings, FlairEmbeddings, DocumentRNNEmbeddings,StackedEmbeddings\n","from flair.models import TextClassifier, sequence_tagger_model\n","from flair.trainers import ModelTrainer\n","from flair.data import Corpus\n","from flair.datasets import ColumnCorpus\n","\n","# tagger = SequenceTagger.load('ner')\n","# define columns\n","columns = {0: 'text', 1: 'ner'}\n","\n","# this is the folder in which train, test and dev files reside\n","data_folder = '/content/gdrive/MyDrive/Entity_extarction_aftab/experi-different-model-aftab/NER project 1'\n","\n","# init a corpus using column format, data folder and the names of the train, dev and test files\n","corpus: Corpus = ColumnCorpus(data_folder, columns,\n","                              train_file='train.txt',\n","                              test_file='test.txt',\n","                              dev_file='dev.txt')\n","print(len(corpus.train))\n","\n","# print the number of Sentences in the test split\n","print(len(corpus.test))\n","\n","# print the number of Sentences in the dev split\n","print(len(corpus.dev))"],"execution_count":15,"outputs":[{"output_type":"stream","text":["2021-05-02 11:54:10,356 Reading data from /content/gdrive/MyDrive/Entity_extarction_aftab/experi-different-model-aftab/NER project 1\n","2021-05-02 11:54:10,357 Train: /content/gdrive/MyDrive/Entity_extarction_aftab/experi-different-model-aftab/NER project 1/train.txt\n","2021-05-02 11:54:10,359 Dev: /content/gdrive/MyDrive/Entity_extarction_aftab/experi-different-model-aftab/NER project 1/dev.txt\n","2021-05-02 11:54:10,360 Test: /content/gdrive/MyDrive/Entity_extarction_aftab/experi-different-model-aftab/NER project 1/test.txt\n","40586\n","3632\n","3546\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"XSVAqqkB3YEb","executionInfo":{"status":"ok","timestamp":1619956468048,"user_tz":-300,"elapsed":11865,"user":{"displayName":"AI Team","photoUrl":"","userId":"17689037188246991143"}},"outputId":"140c645c-f05d-4855-8fbd-6d3319511e99"},"source":["print(corpus.train[0].to_tagged_string('ner'))"],"execution_count":16,"outputs":[{"output_type":"stream","text":["# <-DOCSTART-> the investors New <B-org> his the Earth <B-geo> modest on war , officers prompted Sri <B-per> monthly political former Palestinian <B-org> eliminate 60 . <.>\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"qZiofffqxz2d","executionInfo":{"status":"ok","timestamp":1619956469174,"user_tz":-300,"elapsed":12629,"user":{"displayName":"AI Team","photoUrl":"","userId":"17689037188246991143"}},"outputId":"2f6dbd54-c1e1-45d0-8342-2b49fb155f96"},"source":["label_dict = corpus.make_label_dictionary()"],"execution_count":17,"outputs":[{"output_type":"stream","text":["2021-05-02 11:54:27,601 Computing label dictionary. Progress:\n"],"name":"stdout"},{"output_type":"stream","text":["100%|██████████| 44218/44218 [00:01<00:00, 34236.69it/s]"],"name":"stderr"},{"output_type":"stream","text":["2021-05-02 11:54:28,898 [b'-DOCSTART-', b'O', b'B-org', b'B-geo', b'B-per', b'.', b'B-gpe', b'I-per', b'I-org', b'I-tim', b'B-tim', b'B-art', b'I-geo', b'I-nat', b'B-nat', b'B-eve', b'I-gpe', b'I-art', b'I-eve', b'storm']\n"],"name":"stdout"},{"output_type":"stream","text":["\n"],"name":"stderr"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"FBJ_wRmhxzym","executionInfo":{"status":"ok","timestamp":1619956469175,"user_tz":-300,"elapsed":12181,"user":{"displayName":"AI Team","photoUrl":"","userId":"17689037188246991143"}},"outputId":"33943a0d-a837-422d-da82-d6f00ab78773"},"source":["label_dict.get_items()"],"execution_count":18,"outputs":[{"output_type":"execute_result","data":{"text/plain":["['-DOCSTART-',\n"," 'O',\n"," 'B-org',\n"," 'B-geo',\n"," 'B-per',\n"," '.',\n"," 'B-gpe',\n"," 'I-per',\n"," 'I-org',\n"," 'I-tim',\n"," 'B-tim',\n"," 'B-art',\n"," 'I-geo',\n"," 'I-nat',\n"," 'B-nat',\n"," 'B-eve',\n"," 'I-gpe',\n"," 'I-art',\n"," 'I-eve',\n"," 'storm']"]},"metadata":{"tags":[]},"execution_count":18}]},{"cell_type":"markdown","metadata":{"id":"fyuxG69_r9DZ"},"source":["## Flair with Glove Embedding "]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"kW-tYKfzxzum","executionInfo":{"elapsed":2071090,"status":"ok","timestamp":1619867481007,"user":{"displayName":"aftab anjum","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gjr9EAQIKikAsQFruoHrEH12B72Qi-qbRgbTxCSNg=s64","userId":"09823110291100232267"},"user_tz":-300},"outputId":"6a48c576-4c6f-4461-c283-98f10fb910bd"},"source":["tag_type = 'ner'\n","\n","# 3. make the tag dictionary from the corpus\n","tag_dictionary = corpus.make_tag_dictionary(tag_type=tag_type)\n","\n","# 3. make a list of word embeddings\n","word_embeddings = [WordEmbeddings('glove')]\n","\n","embeddings: StackedEmbeddings = StackedEmbeddings(embeddings=word_embeddings)\n","# 4. initialize document embedding by passing list of word embeddings\n","# Can choose between many RNN types (GRU by default, to change use rnn_type parameter)\n","# document_embeddings = DocumentRNNEmbeddings(word_embeddings, hidden_size=256)\n","from flair.models import SequenceTagger\n","tagger: SequenceTagger = SequenceTagger(hidden_size=256,\n","                                        embeddings=embeddings,\n","                                        tag_dictionary=tag_dictionary,\n","                                        tag_type=tag_type,\n","                                        use_crf=True)\n","# 5. create the text classifier\n","# classifier = TextClassifier(document_embeddings, label_dictionary=label_dict)\n","from flair.trainers import ModelTrainer\n","\n","trainer: ModelTrainer = ModelTrainer(tagger, corpus)\n","\n","# 6. initialize the text classifier trainer\n","# trainer = ModelTrainer(classifier, corpus)\n","trainer.train('resources/taggers/example-ner',\n","              learning_rate=0.1,\n","              mini_batch_size=64,\n","              max_epochs=30)\n"],"execution_count":null,"outputs":[{"output_type":"stream","text":["2021-05-01 09:47:49,881 https://flair.informatik.hu-berlin.de/resources/embeddings/token/glove.gensim.vectors.npy not found in cache, downloading to /tmp/tmp9_jw04ji\n"],"name":"stdout"},{"output_type":"stream","text":["100%|██████████| 160000128/160000128 [00:06<00:00, 26193775.98B/s]"],"name":"stderr"},{"output_type":"stream","text":["2021-05-01 09:47:56,312 copying /tmp/tmp9_jw04ji to cache at /root/.flair/embeddings/glove.gensim.vectors.npy\n"],"name":"stdout"},{"output_type":"stream","text":["\n"],"name":"stderr"},{"output_type":"stream","text":["2021-05-01 09:47:56,922 removing temp file /tmp/tmp9_jw04ji\n","2021-05-01 09:47:58,642 https://flair.informatik.hu-berlin.de/resources/embeddings/token/glove.gensim not found in cache, downloading to /tmp/tmpn0f_ckfn\n"],"name":"stdout"},{"output_type":"stream","text":["100%|██████████| 21494764/21494764 [00:01<00:00, 15248645.74B/s]"],"name":"stderr"},{"output_type":"stream","text":["2021-05-01 09:48:00,386 copying /tmp/tmpn0f_ckfn to cache at /root/.flair/embeddings/glove.gensim\n","2021-05-01 09:48:00,410 removing temp file /tmp/tmpn0f_ckfn\n"],"name":"stdout"},{"output_type":"stream","text":["\n"],"name":"stderr"},{"output_type":"stream","text":["2021-05-01 09:48:04,626 ----------------------------------------------------------------------------------------------------\n","2021-05-01 09:48:04,628 Model: \"SequenceTagger(\n","  (embeddings): StackedEmbeddings(\n","    (list_embedding_0): WordEmbeddings('glove')\n","  )\n","  (word_dropout): WordDropout(p=0.05)\n","  (locked_dropout): LockedDropout(p=0.5)\n","  (embedding2nn): Linear(in_features=100, out_features=100, bias=True)\n","  (rnn): LSTM(100, 256, batch_first=True, bidirectional=True)\n","  (linear): Linear(in_features=512, out_features=23, bias=True)\n","  (beta): 1.0\n","  (weights): None\n","  (weight_tensor) None\n",")\"\n","2021-05-01 09:48:04,631 ----------------------------------------------------------------------------------------------------\n","2021-05-01 09:48:04,633 Corpus: \"Corpus: 40586 train + 3546 dev + 3632 test sentences\"\n","2021-05-01 09:48:04,635 ----------------------------------------------------------------------------------------------------\n","2021-05-01 09:48:04,637 Parameters:\n","2021-05-01 09:48:04,638  - learning_rate: \"0.1\"\n","2021-05-01 09:48:04,639  - mini_batch_size: \"64\"\n","2021-05-01 09:48:04,640  - patience: \"3\"\n","2021-05-01 09:48:04,665  - anneal_factor: \"0.5\"\n","2021-05-01 09:48:04,667  - max_epochs: \"30\"\n","2021-05-01 09:48:04,668  - shuffle: \"True\"\n","2021-05-01 09:48:04,669  - train_with_dev: \"False\"\n","2021-05-01 09:48:04,673  - batch_growth_annealing: \"False\"\n","2021-05-01 09:48:04,674 ----------------------------------------------------------------------------------------------------\n","2021-05-01 09:48:04,675 Model training base path: \"resources/taggers/example-ner\"\n","2021-05-01 09:48:04,676 ----------------------------------------------------------------------------------------------------\n","2021-05-01 09:48:04,678 Device: cuda:0\n","2021-05-01 09:48:04,679 ----------------------------------------------------------------------------------------------------\n","2021-05-01 09:48:04,680 Embeddings storage mode: cpu\n","2021-05-01 09:48:04,683 ----------------------------------------------------------------------------------------------------\n","2021-05-01 09:48:28,658 epoch 1 - iter 63/635 - loss 21.26743364 - samples/sec: 168.21 - lr: 0.100000\n","2021-05-01 09:48:46,740 epoch 1 - iter 126/635 - loss 18.01075174 - samples/sec: 223.06 - lr: 0.100000\n","2021-05-01 09:49:04,481 epoch 1 - iter 189/635 - loss 16.12813862 - samples/sec: 227.35 - lr: 0.100000\n","2021-05-01 09:49:22,719 epoch 1 - iter 252/635 - loss 14.93852340 - samples/sec: 221.15 - lr: 0.100000\n","2021-05-01 09:49:41,602 epoch 1 - iter 315/635 - loss 14.07099787 - samples/sec: 213.59 - lr: 0.100000\n","2021-05-01 09:50:00,020 epoch 1 - iter 378/635 - loss 13.42332529 - samples/sec: 219.00 - lr: 0.100000\n","2021-05-01 09:50:18,173 epoch 1 - iter 441/635 - loss 12.94262471 - samples/sec: 222.18 - lr: 0.100000\n","2021-05-01 09:50:36,522 epoch 1 - iter 504/635 - loss 12.53117247 - samples/sec: 219.82 - lr: 0.100000\n","2021-05-01 09:50:56,992 epoch 1 - iter 567/635 - loss 12.21642818 - samples/sec: 197.03 - lr: 0.100000\n","2021-05-01 09:51:14,390 epoch 1 - iter 630/635 - loss 11.90318027 - samples/sec: 231.84 - lr: 0.100000\n","2021-05-01 09:51:15,580 ----------------------------------------------------------------------------------------------------\n","2021-05-01 09:51:15,581 EPOCH 1 done: loss 11.8793 - lr 0.1000000\n","2021-05-01 09:51:24,625 DEV : loss 7.366448879241943 - score 0.7271\n","2021-05-01 09:51:24,781 BAD EPOCHS (no improvement): 0\n","saving best model\n","2021-05-01 09:51:27,645 ----------------------------------------------------------------------------------------------------\n","2021-05-01 09:51:43,202 epoch 2 - iter 63/635 - loss 9.19532348 - samples/sec: 259.40 - lr: 0.100000\n","2021-05-01 09:51:58,398 epoch 2 - iter 126/635 - loss 9.02768566 - samples/sec: 265.46 - lr: 0.100000\n","2021-05-01 09:52:13,510 epoch 2 - iter 189/635 - loss 8.93487201 - samples/sec: 266.96 - lr: 0.100000\n","2021-05-01 09:52:29,194 epoch 2 - iter 252/635 - loss 8.93636538 - samples/sec: 257.19 - lr: 0.100000\n","2021-05-01 09:52:44,925 epoch 2 - iter 315/635 - loss 8.87767843 - samples/sec: 256.42 - lr: 0.100000\n","2021-05-01 09:53:00,262 epoch 2 - iter 378/635 - loss 8.83492457 - samples/sec: 263.00 - lr: 0.100000\n","2021-05-01 09:53:15,834 epoch 2 - iter 441/635 - loss 8.75423255 - samples/sec: 259.05 - lr: 0.100000\n","2021-05-01 09:53:31,732 epoch 2 - iter 504/635 - loss 8.72743211 - samples/sec: 253.74 - lr: 0.100000\n","2021-05-01 09:53:47,092 epoch 2 - iter 567/635 - loss 8.65774755 - samples/sec: 262.61 - lr: 0.100000\n","2021-05-01 09:54:02,821 epoch 2 - iter 630/635 - loss 8.62605223 - samples/sec: 256.49 - lr: 0.100000\n","2021-05-01 09:54:03,911 ----------------------------------------------------------------------------------------------------\n","2021-05-01 09:54:03,913 EPOCH 2 done: loss 8.6194 - lr 0.1000000\n","2021-05-01 09:54:10,656 DEV : loss 6.434582710266113 - score 0.7567\n","2021-05-01 09:54:10,812 BAD EPOCHS (no improvement): 0\n","saving best model\n","2021-05-01 09:54:13,384 ----------------------------------------------------------------------------------------------------\n","2021-05-01 09:54:29,129 epoch 3 - iter 63/635 - loss 8.17806977 - samples/sec: 256.30 - lr: 0.100000\n","2021-05-01 09:54:44,039 epoch 3 - iter 126/635 - loss 7.94957992 - samples/sec: 270.56 - lr: 0.100000\n","2021-05-01 09:54:59,657 epoch 3 - iter 189/635 - loss 8.03038521 - samples/sec: 258.27 - lr: 0.100000\n","2021-05-01 09:55:15,012 epoch 3 - iter 252/635 - loss 7.93166164 - samples/sec: 262.71 - lr: 0.100000\n","2021-05-01 09:55:30,776 epoch 3 - iter 315/635 - loss 7.96736019 - samples/sec: 255.89 - lr: 0.100000\n","2021-05-01 09:55:46,012 epoch 3 - iter 378/635 - loss 7.95525610 - samples/sec: 264.78 - lr: 0.100000\n","2021-05-01 09:56:01,807 epoch 3 - iter 441/635 - loss 7.94020704 - samples/sec: 255.40 - lr: 0.100000\n","2021-05-01 09:56:18,164 epoch 3 - iter 504/635 - loss 7.94151140 - samples/sec: 246.61 - lr: 0.100000\n","2021-05-01 09:56:34,024 epoch 3 - iter 567/635 - loss 7.93108668 - samples/sec: 254.34 - lr: 0.100000\n","2021-05-01 09:56:49,603 epoch 3 - iter 630/635 - loss 7.91430588 - samples/sec: 258.92 - lr: 0.100000\n","2021-05-01 09:56:50,808 ----------------------------------------------------------------------------------------------------\n","2021-05-01 09:56:50,810 EPOCH 3 done: loss 7.9226 - lr 0.1000000\n","2021-05-01 09:56:57,509 DEV : loss 5.904755115509033 - score 0.7692\n","2021-05-01 09:56:57,669 BAD EPOCHS (no improvement): 0\n","saving best model\n","2021-05-01 09:57:00,219 ----------------------------------------------------------------------------------------------------\n","2021-05-01 09:57:15,916 epoch 4 - iter 63/635 - loss 7.77233637 - samples/sec: 257.10 - lr: 0.100000\n","2021-05-01 09:57:31,242 epoch 4 - iter 126/635 - loss 7.66658054 - samples/sec: 263.20 - lr: 0.100000\n","2021-05-01 09:57:47,609 epoch 4 - iter 189/635 - loss 7.71033600 - samples/sec: 246.46 - lr: 0.100000\n","2021-05-01 09:58:02,580 epoch 4 - iter 252/635 - loss 7.75756420 - samples/sec: 269.44 - lr: 0.100000\n","2021-05-01 09:58:18,400 epoch 4 - iter 315/635 - loss 7.74022578 - samples/sec: 255.01 - lr: 0.100000\n","2021-05-01 09:58:33,744 epoch 4 - iter 378/635 - loss 7.68808582 - samples/sec: 262.87 - lr: 0.100000\n","2021-05-01 09:58:49,542 epoch 4 - iter 441/635 - loss 7.65144336 - samples/sec: 255.34 - lr: 0.100000\n","2021-05-01 09:59:04,962 epoch 4 - iter 504/635 - loss 7.61940231 - samples/sec: 261.60 - lr: 0.100000\n","2021-05-01 09:59:20,948 epoch 4 - iter 567/635 - loss 7.60278677 - samples/sec: 252.34 - lr: 0.100000\n","2021-05-01 09:59:36,229 epoch 4 - iter 630/635 - loss 7.57739148 - samples/sec: 263.97 - lr: 0.100000\n","2021-05-01 09:59:37,292 ----------------------------------------------------------------------------------------------------\n","2021-05-01 09:59:37,293 EPOCH 4 done: loss 7.5817 - lr 0.1000000\n","2021-05-01 09:59:44,067 DEV : loss 5.7200212478637695 - score 0.7736\n","2021-05-01 09:59:44,222 BAD EPOCHS (no improvement): 0\n","saving best model\n","2021-05-01 09:59:46,792 ----------------------------------------------------------------------------------------------------\n","2021-05-01 10:00:02,155 epoch 5 - iter 63/635 - loss 7.37100314 - samples/sec: 262.70 - lr: 0.100000\n","2021-05-01 10:00:18,096 epoch 5 - iter 126/635 - loss 7.46960698 - samples/sec: 253.05 - lr: 0.100000\n","2021-05-01 10:00:34,003 epoch 5 - iter 189/635 - loss 7.46362035 - samples/sec: 253.59 - lr: 0.100000\n","2021-05-01 10:00:49,862 epoch 5 - iter 252/635 - loss 7.40972693 - samples/sec: 254.36 - lr: 0.100000\n","2021-05-01 10:01:05,536 epoch 5 - iter 315/635 - loss 7.36456993 - samples/sec: 257.35 - lr: 0.100000\n","2021-05-01 10:01:21,368 epoch 5 - iter 378/635 - loss 7.38455056 - samples/sec: 254.79 - lr: 0.100000\n","2021-05-01 10:01:37,192 epoch 5 - iter 441/635 - loss 7.38582248 - samples/sec: 254.93 - lr: 0.100000\n","2021-05-01 10:01:52,866 epoch 5 - iter 504/635 - loss 7.36540594 - samples/sec: 257.35 - lr: 0.100000\n","2021-05-01 10:02:08,575 epoch 5 - iter 567/635 - loss 7.36099539 - samples/sec: 256.79 - lr: 0.100000\n","2021-05-01 10:02:23,623 epoch 5 - iter 630/635 - loss 7.34181970 - samples/sec: 268.07 - lr: 0.100000\n","2021-05-01 10:02:24,788 ----------------------------------------------------------------------------------------------------\n","2021-05-01 10:02:24,789 EPOCH 5 done: loss 7.3459 - lr 0.1000000\n","2021-05-01 10:02:31,530 DEV : loss 5.672123908996582 - score 0.7747\n","2021-05-01 10:02:31,687 BAD EPOCHS (no improvement): 0\n","saving best model\n","2021-05-01 10:02:34,248 ----------------------------------------------------------------------------------------------------\n","2021-05-01 10:02:50,354 epoch 6 - iter 63/635 - loss 7.16815827 - samples/sec: 250.53 - lr: 0.100000\n","2021-05-01 10:03:06,395 epoch 6 - iter 126/635 - loss 7.14512359 - samples/sec: 251.47 - lr: 0.100000\n","2021-05-01 10:03:21,691 epoch 6 - iter 189/635 - loss 7.17015076 - samples/sec: 263.72 - lr: 0.100000\n","2021-05-01 10:03:39,194 epoch 6 - iter 252/635 - loss 7.11294866 - samples/sec: 230.48 - lr: 0.100000\n","2021-05-01 10:03:55,059 epoch 6 - iter 315/635 - loss 7.15048209 - samples/sec: 254.26 - lr: 0.100000\n","2021-05-01 10:04:10,864 epoch 6 - iter 378/635 - loss 7.17337766 - samples/sec: 255.22 - lr: 0.100000\n","2021-05-01 10:04:26,417 epoch 6 - iter 441/635 - loss 7.18025291 - samples/sec: 259.35 - lr: 0.100000\n","2021-05-01 10:04:42,374 epoch 6 - iter 504/635 - loss 7.19342745 - samples/sec: 252.79 - lr: 0.100000\n","2021-05-01 10:04:58,214 epoch 6 - iter 567/635 - loss 7.21081907 - samples/sec: 254.67 - lr: 0.100000\n","2021-05-01 10:05:13,551 epoch 6 - iter 630/635 - loss 7.18534952 - samples/sec: 263.04 - lr: 0.100000\n","2021-05-01 10:05:14,561 ----------------------------------------------------------------------------------------------------\n","2021-05-01 10:05:14,562 EPOCH 6 done: loss 7.1829 - lr 0.1000000\n","2021-05-01 10:05:21,292 DEV : loss 5.504039287567139 - score 0.7762\n","2021-05-01 10:05:21,452 BAD EPOCHS (no improvement): 0\n","saving best model\n","2021-05-01 10:05:24,036 ----------------------------------------------------------------------------------------------------\n","2021-05-01 10:05:39,863 epoch 7 - iter 63/635 - loss 7.13034061 - samples/sec: 254.98 - lr: 0.100000\n","2021-05-01 10:05:55,201 epoch 7 - iter 126/635 - loss 7.13561720 - samples/sec: 263.00 - lr: 0.100000\n","2021-05-01 10:06:11,042 epoch 7 - iter 189/635 - loss 7.05152697 - samples/sec: 254.66 - lr: 0.100000\n","2021-05-01 10:06:26,279 epoch 7 - iter 252/635 - loss 7.07271256 - samples/sec: 264.75 - lr: 0.100000\n","2021-05-01 10:06:41,861 epoch 7 - iter 315/635 - loss 7.09355155 - samples/sec: 258.87 - lr: 0.100000\n","2021-05-01 10:06:57,472 epoch 7 - iter 378/635 - loss 7.09907277 - samples/sec: 258.38 - lr: 0.100000\n","2021-05-01 10:07:12,491 epoch 7 - iter 441/635 - loss 7.06488458 - samples/sec: 268.59 - lr: 0.100000\n","2021-05-01 10:07:28,191 epoch 7 - iter 504/635 - loss 7.04487719 - samples/sec: 256.92 - lr: 0.100000\n","2021-05-01 10:07:44,418 epoch 7 - iter 567/635 - loss 7.06062470 - samples/sec: 248.60 - lr: 0.100000\n","2021-05-01 10:08:00,221 epoch 7 - iter 630/635 - loss 7.06289374 - samples/sec: 255.25 - lr: 0.100000\n","2021-05-01 10:08:01,293 ----------------------------------------------------------------------------------------------------\n","2021-05-01 10:08:01,294 EPOCH 7 done: loss 7.0613 - lr 0.1000000\n","2021-05-01 10:08:08,033 DEV : loss 5.449540615081787 - score 0.7813\n","2021-05-01 10:08:08,187 BAD EPOCHS (no improvement): 0\n","saving best model\n","2021-05-01 10:08:10,888 ----------------------------------------------------------------------------------------------------\n","2021-05-01 10:08:26,875 epoch 8 - iter 63/635 - loss 7.06772969 - samples/sec: 252.69 - lr: 0.100000\n","2021-05-01 10:08:42,938 epoch 8 - iter 126/635 - loss 7.09661273 - samples/sec: 251.12 - lr: 0.100000\n","2021-05-01 10:08:58,653 epoch 8 - iter 189/635 - loss 7.08187121 - samples/sec: 256.68 - lr: 0.100000\n","2021-05-01 10:09:14,078 epoch 8 - iter 252/635 - loss 7.03759181 - samples/sec: 261.55 - lr: 0.100000\n","2021-05-01 10:09:29,782 epoch 8 - iter 315/635 - loss 7.02693782 - samples/sec: 256.86 - lr: 0.100000\n","2021-05-01 10:09:45,592 epoch 8 - iter 378/635 - loss 7.02210523 - samples/sec: 255.15 - lr: 0.100000\n","2021-05-01 10:10:01,014 epoch 8 - iter 441/635 - loss 7.00459875 - samples/sec: 261.56 - lr: 0.100000\n","2021-05-01 10:10:17,144 epoch 8 - iter 504/635 - loss 6.99826166 - samples/sec: 250.07 - lr: 0.100000\n","2021-05-01 10:10:32,225 epoch 8 - iter 567/635 - loss 6.97929943 - samples/sec: 267.49 - lr: 0.100000\n","2021-05-01 10:10:47,444 epoch 8 - iter 630/635 - loss 6.97009305 - samples/sec: 265.04 - lr: 0.100000\n","2021-05-01 10:10:48,431 ----------------------------------------------------------------------------------------------------\n","2021-05-01 10:10:48,432 EPOCH 8 done: loss 6.9612 - lr 0.1000000\n","2021-05-01 10:10:55,233 DEV : loss 5.392806053161621 - score 0.7829\n","2021-05-01 10:10:55,395 BAD EPOCHS (no improvement): 0\n","saving best model\n","2021-05-01 10:10:57,990 ----------------------------------------------------------------------------------------------------\n","2021-05-01 10:11:13,924 epoch 9 - iter 63/635 - loss 6.79647904 - samples/sec: 253.23 - lr: 0.100000\n","2021-05-01 10:11:29,375 epoch 9 - iter 126/635 - loss 6.77427597 - samples/sec: 261.06 - lr: 0.100000\n","2021-05-01 10:11:44,618 epoch 9 - iter 189/635 - loss 6.83906012 - samples/sec: 264.64 - lr: 0.100000\n","2021-05-01 10:12:00,368 epoch 9 - iter 252/635 - loss 6.82912680 - samples/sec: 256.11 - lr: 0.100000\n","2021-05-01 10:12:16,500 epoch 9 - iter 315/635 - loss 6.83788876 - samples/sec: 250.06 - lr: 0.100000\n","2021-05-01 10:12:32,629 epoch 9 - iter 378/635 - loss 6.86740473 - samples/sec: 250.10 - lr: 0.100000\n","2021-05-01 10:12:47,722 epoch 9 - iter 441/635 - loss 6.84233744 - samples/sec: 267.25 - lr: 0.100000\n","2021-05-01 10:13:03,173 epoch 9 - iter 504/635 - loss 6.86424219 - samples/sec: 261.07 - lr: 0.100000\n","2021-05-01 10:13:18,262 epoch 9 - iter 567/635 - loss 6.85422958 - samples/sec: 267.36 - lr: 0.100000\n","2021-05-01 10:13:34,006 epoch 9 - iter 630/635 - loss 6.84385128 - samples/sec: 256.21 - lr: 0.100000\n","2021-05-01 10:13:35,132 ----------------------------------------------------------------------------------------------------\n","2021-05-01 10:13:35,133 EPOCH 9 done: loss 6.8425 - lr 0.1000000\n","2021-05-01 10:13:41,909 DEV : loss 5.37333345413208 - score 0.7845\n","2021-05-01 10:13:42,068 BAD EPOCHS (no improvement): 0\n","saving best model\n","2021-05-01 10:13:44,694 ----------------------------------------------------------------------------------------------------\n","2021-05-01 10:14:00,746 epoch 10 - iter 63/635 - loss 6.78437001 - samples/sec: 251.39 - lr: 0.100000\n","2021-05-01 10:14:16,683 epoch 10 - iter 126/635 - loss 6.85761512 - samples/sec: 253.12 - lr: 0.100000\n","2021-05-01 10:14:32,122 epoch 10 - iter 189/635 - loss 6.81148945 - samples/sec: 261.29 - lr: 0.100000\n","2021-05-01 10:14:48,035 epoch 10 - iter 252/635 - loss 6.81839838 - samples/sec: 253.51 - lr: 0.100000\n","2021-05-01 10:15:03,539 epoch 10 - iter 315/635 - loss 6.76933404 - samples/sec: 260.21 - lr: 0.100000\n","2021-05-01 10:15:19,291 epoch 10 - iter 378/635 - loss 6.80336145 - samples/sec: 256.10 - lr: 0.100000\n","2021-05-01 10:15:35,547 epoch 10 - iter 441/635 - loss 6.82277719 - samples/sec: 248.13 - lr: 0.100000\n","2021-05-01 10:15:50,380 epoch 10 - iter 504/635 - loss 6.79318807 - samples/sec: 271.95 - lr: 0.100000\n","2021-05-01 10:16:05,580 epoch 10 - iter 567/635 - loss 6.80564742 - samples/sec: 265.37 - lr: 0.100000\n","2021-05-01 10:16:20,707 epoch 10 - iter 630/635 - loss 6.78512846 - samples/sec: 266.66 - lr: 0.100000\n","2021-05-01 10:16:21,842 ----------------------------------------------------------------------------------------------------\n","2021-05-01 10:16:21,844 EPOCH 10 done: loss 6.7848 - lr 0.1000000\n","2021-05-01 10:16:28,552 DEV : loss 5.257050514221191 - score 0.7893\n","2021-05-01 10:16:28,714 BAD EPOCHS (no improvement): 0\n","saving best model\n","2021-05-01 10:16:31,230 ----------------------------------------------------------------------------------------------------\n","2021-05-01 10:16:46,754 epoch 11 - iter 63/635 - loss 6.69169434 - samples/sec: 259.90 - lr: 0.100000\n","2021-05-01 10:17:01,811 epoch 11 - iter 126/635 - loss 6.72893423 - samples/sec: 267.92 - lr: 0.100000\n","2021-05-01 10:17:16,958 epoch 11 - iter 189/635 - loss 6.73030855 - samples/sec: 266.32 - lr: 0.100000\n","2021-05-01 10:17:32,988 epoch 11 - iter 252/635 - loss 6.73836980 - samples/sec: 251.63 - lr: 0.100000\n","2021-05-01 10:17:48,341 epoch 11 - iter 315/635 - loss 6.72157492 - samples/sec: 262.73 - lr: 0.100000\n","2021-05-01 10:18:04,035 epoch 11 - iter 378/635 - loss 6.71164477 - samples/sec: 257.02 - lr: 0.100000\n","2021-05-01 10:18:20,088 epoch 11 - iter 441/635 - loss 6.73153558 - samples/sec: 251.28 - lr: 0.100000\n","2021-05-01 10:18:35,360 epoch 11 - iter 504/635 - loss 6.72283103 - samples/sec: 264.13 - lr: 0.100000\n","2021-05-01 10:18:50,494 epoch 11 - iter 567/635 - loss 6.71842339 - samples/sec: 266.54 - lr: 0.100000\n","2021-05-01 10:19:06,493 epoch 11 - iter 630/635 - loss 6.72597034 - samples/sec: 252.13 - lr: 0.100000\n","2021-05-01 10:19:07,558 ----------------------------------------------------------------------------------------------------\n","2021-05-01 10:19:07,560 EPOCH 11 done: loss 6.7222 - lr 0.1000000\n","2021-05-01 10:19:14,235 DEV : loss 5.211832046508789 - score 0.7894\n","2021-05-01 10:19:14,390 BAD EPOCHS (no improvement): 0\n","saving best model\n","2021-05-01 10:19:16,928 ----------------------------------------------------------------------------------------------------\n","2021-05-01 10:19:32,121 epoch 12 - iter 63/635 - loss 6.58996119 - samples/sec: 265.69 - lr: 0.100000\n","2021-05-01 10:19:47,557 epoch 12 - iter 126/635 - loss 6.57930310 - samples/sec: 261.32 - lr: 0.100000\n","2021-05-01 10:20:02,957 epoch 12 - iter 189/635 - loss 6.64437790 - samples/sec: 261.94 - lr: 0.100000\n","2021-05-01 10:20:18,703 epoch 12 - iter 252/635 - loss 6.64539697 - samples/sec: 256.24 - lr: 0.100000\n","2021-05-01 10:20:34,316 epoch 12 - iter 315/635 - loss 6.64690908 - samples/sec: 258.35 - lr: 0.100000\n","2021-05-01 10:20:49,771 epoch 12 - iter 378/635 - loss 6.66704860 - samples/sec: 261.00 - lr: 0.100000\n","2021-05-01 10:21:05,436 epoch 12 - iter 441/635 - loss 6.67427743 - samples/sec: 257.51 - lr: 0.100000\n","2021-05-01 10:21:21,074 epoch 12 - iter 504/635 - loss 6.67182125 - samples/sec: 257.93 - lr: 0.100000\n","2021-05-01 10:21:36,494 epoch 12 - iter 567/635 - loss 6.68654761 - samples/sec: 261.61 - lr: 0.100000\n","2021-05-01 10:21:51,919 epoch 12 - iter 630/635 - loss 6.67330936 - samples/sec: 261.52 - lr: 0.100000\n","2021-05-01 10:21:53,124 ----------------------------------------------------------------------------------------------------\n","2021-05-01 10:21:53,125 EPOCH 12 done: loss 6.6734 - lr 0.1000000\n","2021-05-01 10:21:59,788 DEV : loss 5.187159538269043 - score 0.7907\n","2021-05-01 10:21:59,942 BAD EPOCHS (no improvement): 0\n","saving best model\n","2021-05-01 10:22:02,456 ----------------------------------------------------------------------------------------------------\n","2021-05-01 10:22:17,409 epoch 13 - iter 63/635 - loss 6.63775704 - samples/sec: 269.89 - lr: 0.100000\n","2021-05-01 10:22:32,934 epoch 13 - iter 126/635 - loss 6.83456720 - samples/sec: 259.81 - lr: 0.100000\n","2021-05-01 10:22:49,073 epoch 13 - iter 189/635 - loss 6.77175775 - samples/sec: 249.95 - lr: 0.100000\n","2021-05-01 10:23:04,419 epoch 13 - iter 252/635 - loss 6.67224250 - samples/sec: 262.86 - lr: 0.100000\n","2021-05-01 10:23:20,478 epoch 13 - iter 315/635 - loss 6.63998533 - samples/sec: 251.18 - lr: 0.100000\n","2021-05-01 10:23:35,756 epoch 13 - iter 378/635 - loss 6.64244890 - samples/sec: 264.01 - lr: 0.100000\n","2021-05-01 10:23:51,283 epoch 13 - iter 441/635 - loss 6.64379074 - samples/sec: 259.80 - lr: 0.100000\n","2021-05-01 10:24:06,694 epoch 13 - iter 504/635 - loss 6.63869102 - samples/sec: 261.76 - lr: 0.100000\n","2021-05-01 10:24:22,404 epoch 13 - iter 567/635 - loss 6.64130962 - samples/sec: 256.76 - lr: 0.100000\n","2021-05-01 10:24:37,636 epoch 13 - iter 630/635 - loss 6.63697940 - samples/sec: 264.82 - lr: 0.100000\n","2021-05-01 10:24:38,710 ----------------------------------------------------------------------------------------------------\n","2021-05-01 10:24:38,712 EPOCH 13 done: loss 6.6345 - lr 0.1000000\n","2021-05-01 10:24:47,650 DEV : loss 5.126097202301025 - score 0.791\n","2021-05-01 10:24:47,806 BAD EPOCHS (no improvement): 0\n","saving best model\n","2021-05-01 10:24:50,312 ----------------------------------------------------------------------------------------------------\n","2021-05-01 10:25:06,237 epoch 14 - iter 63/635 - loss 6.59953917 - samples/sec: 253.38 - lr: 0.100000\n","2021-05-01 10:25:21,947 epoch 14 - iter 126/635 - loss 6.63473114 - samples/sec: 256.76 - lr: 0.100000\n","2021-05-01 10:25:37,260 epoch 14 - iter 189/635 - loss 6.55893877 - samples/sec: 263.43 - lr: 0.100000\n","2021-05-01 10:25:52,605 epoch 14 - iter 252/635 - loss 6.57898402 - samples/sec: 262.86 - lr: 0.100000\n","2021-05-01 10:26:08,084 epoch 14 - iter 315/635 - loss 6.56503480 - samples/sec: 260.60 - lr: 0.100000\n","2021-05-01 10:26:23,341 epoch 14 - iter 378/635 - loss 6.55326265 - samples/sec: 264.38 - lr: 0.100000\n","2021-05-01 10:26:38,627 epoch 14 - iter 441/635 - loss 6.56482460 - samples/sec: 263.89 - lr: 0.100000\n","2021-05-01 10:26:54,668 epoch 14 - iter 504/635 - loss 6.56116412 - samples/sec: 251.46 - lr: 0.100000\n","2021-05-01 10:27:10,134 epoch 14 - iter 567/635 - loss 6.58510541 - samples/sec: 260.82 - lr: 0.100000\n","2021-05-01 10:27:25,619 epoch 14 - iter 630/635 - loss 6.57993238 - samples/sec: 260.50 - lr: 0.100000\n","2021-05-01 10:27:26,904 ----------------------------------------------------------------------------------------------------\n","2021-05-01 10:27:26,905 EPOCH 14 done: loss 6.5822 - lr 0.1000000\n","2021-05-01 10:27:33,593 DEV : loss 5.1455302238464355 - score 0.7878\n","2021-05-01 10:27:33,754 BAD EPOCHS (no improvement): 1\n","2021-05-01 10:27:33,755 ----------------------------------------------------------------------------------------------------\n","2021-05-01 10:27:49,526 epoch 15 - iter 63/635 - loss 6.51017706 - samples/sec: 255.86 - lr: 0.100000\n","2021-05-01 10:28:04,667 epoch 15 - iter 126/635 - loss 6.46077267 - samples/sec: 266.43 - lr: 0.100000\n","2021-05-01 10:28:20,325 epoch 15 - iter 189/635 - loss 6.46332459 - samples/sec: 257.63 - lr: 0.100000\n","2021-05-01 10:28:35,937 epoch 15 - iter 252/635 - loss 6.51932065 - samples/sec: 258.37 - lr: 0.100000\n","2021-05-01 10:28:50,977 epoch 15 - iter 315/635 - loss 6.53165675 - samples/sec: 268.20 - lr: 0.100000\n","2021-05-01 10:29:06,471 epoch 15 - iter 378/635 - loss 6.54846599 - samples/sec: 260.38 - lr: 0.100000\n","2021-05-01 10:29:21,831 epoch 15 - iter 441/635 - loss 6.56520200 - samples/sec: 262.61 - lr: 0.100000\n","2021-05-01 10:29:37,487 epoch 15 - iter 504/635 - loss 6.55711295 - samples/sec: 257.66 - lr: 0.100000\n","2021-05-01 10:29:52,705 epoch 15 - iter 567/635 - loss 6.55125890 - samples/sec: 265.07 - lr: 0.100000\n","2021-05-01 10:30:08,317 epoch 15 - iter 630/635 - loss 6.54624290 - samples/sec: 258.38 - lr: 0.100000\n","2021-05-01 10:30:09,496 ----------------------------------------------------------------------------------------------------\n","2021-05-01 10:30:09,497 EPOCH 15 done: loss 6.5449 - lr 0.1000000\n","2021-05-01 10:30:16,149 DEV : loss 5.105213165283203 - score 0.7913\n","2021-05-01 10:30:16,306 BAD EPOCHS (no improvement): 0\n","saving best model\n","2021-05-01 10:30:18,801 ----------------------------------------------------------------------------------------------------\n","2021-05-01 10:30:34,384 epoch 16 - iter 63/635 - loss 6.45197469 - samples/sec: 258.93 - lr: 0.100000\n","2021-05-01 10:30:49,977 epoch 16 - iter 126/635 - loss 6.53207186 - samples/sec: 258.68 - lr: 0.100000\n","2021-05-01 10:31:04,997 epoch 16 - iter 189/635 - loss 6.52719236 - samples/sec: 268.61 - lr: 0.100000\n","2021-05-01 10:31:20,302 epoch 16 - iter 252/635 - loss 6.52022553 - samples/sec: 263.55 - lr: 0.100000\n","2021-05-01 10:31:35,473 epoch 16 - iter 315/635 - loss 6.52750880 - samples/sec: 265.88 - lr: 0.100000\n","2021-05-01 10:31:51,542 epoch 16 - iter 378/635 - loss 6.52232238 - samples/sec: 251.06 - lr: 0.100000\n","2021-05-01 10:32:06,818 epoch 16 - iter 441/635 - loss 6.53065154 - samples/sec: 264.08 - lr: 0.100000\n","2021-05-01 10:32:22,826 epoch 16 - iter 504/635 - loss 6.54384798 - samples/sec: 251.99 - lr: 0.100000\n","2021-05-01 10:32:38,346 epoch 16 - iter 567/635 - loss 6.54644080 - samples/sec: 259.91 - lr: 0.100000\n","2021-05-01 10:32:53,741 epoch 16 - iter 630/635 - loss 6.53497222 - samples/sec: 262.01 - lr: 0.100000\n","2021-05-01 10:32:54,733 ----------------------------------------------------------------------------------------------------\n","2021-05-01 10:32:54,736 EPOCH 16 done: loss 6.5210 - lr 0.1000000\n","2021-05-01 10:33:01,344 DEV : loss 5.104139804840088 - score 0.7908\n","2021-05-01 10:33:01,505 BAD EPOCHS (no improvement): 1\n","2021-05-01 10:33:01,506 ----------------------------------------------------------------------------------------------------\n","2021-05-01 10:33:17,259 epoch 17 - iter 63/635 - loss 6.52476284 - samples/sec: 256.17 - lr: 0.100000\n","2021-05-01 10:33:32,500 epoch 17 - iter 126/635 - loss 6.46315745 - samples/sec: 264.67 - lr: 0.100000\n","2021-05-01 10:33:47,158 epoch 17 - iter 189/635 - loss 6.33627364 - samples/sec: 275.20 - lr: 0.100000\n","2021-05-01 10:34:02,683 epoch 17 - iter 252/635 - loss 6.36024354 - samples/sec: 259.83 - lr: 0.100000\n","2021-05-01 10:34:18,806 epoch 17 - iter 315/635 - loss 6.40457369 - samples/sec: 250.18 - lr: 0.100000\n","2021-05-01 10:34:34,447 epoch 17 - iter 378/635 - loss 6.43661551 - samples/sec: 257.90 - lr: 0.100000\n","2021-05-01 10:34:50,172 epoch 17 - iter 441/635 - loss 6.46950726 - samples/sec: 256.53 - lr: 0.100000\n","2021-05-01 10:35:06,498 epoch 17 - iter 504/635 - loss 6.49114073 - samples/sec: 247.08 - lr: 0.100000\n","2021-05-01 10:35:22,383 epoch 17 - iter 567/635 - loss 6.48924200 - samples/sec: 253.94 - lr: 0.100000\n","2021-05-01 10:35:38,039 epoch 17 - iter 630/635 - loss 6.47261048 - samples/sec: 257.64 - lr: 0.100000\n","2021-05-01 10:35:39,085 ----------------------------------------------------------------------------------------------------\n","2021-05-01 10:35:39,086 EPOCH 17 done: loss 6.4646 - lr 0.1000000\n","2021-05-01 10:35:45,919 DEV : loss 5.07221794128418 - score 0.7909\n","2021-05-01 10:35:46,074 BAD EPOCHS (no improvement): 2\n","2021-05-01 10:35:46,075 ----------------------------------------------------------------------------------------------------\n","2021-05-01 10:36:01,380 epoch 18 - iter 63/635 - loss 6.49954128 - samples/sec: 263.64 - lr: 0.100000\n","2021-05-01 10:36:17,209 epoch 18 - iter 126/635 - loss 6.52310764 - samples/sec: 254.86 - lr: 0.100000\n","2021-05-01 10:36:33,435 epoch 18 - iter 189/635 - loss 6.53314100 - samples/sec: 248.60 - lr: 0.100000\n","2021-05-01 10:36:49,043 epoch 18 - iter 252/635 - loss 6.51772286 - samples/sec: 258.43 - lr: 0.100000\n","2021-05-01 10:37:04,508 epoch 18 - iter 315/635 - loss 6.53507289 - samples/sec: 260.85 - lr: 0.100000\n","2021-05-01 10:37:20,146 epoch 18 - iter 378/635 - loss 6.51330446 - samples/sec: 257.94 - lr: 0.100000\n","2021-05-01 10:37:35,530 epoch 18 - iter 441/635 - loss 6.48726266 - samples/sec: 262.24 - lr: 0.100000\n","2021-05-01 10:37:50,936 epoch 18 - iter 504/635 - loss 6.47223396 - samples/sec: 261.84 - lr: 0.100000\n","2021-05-01 10:38:06,102 epoch 18 - iter 567/635 - loss 6.47622958 - samples/sec: 265.98 - lr: 0.100000\n","2021-05-01 10:38:21,143 epoch 18 - iter 630/635 - loss 6.46437245 - samples/sec: 268.19 - lr: 0.100000\n","2021-05-01 10:38:22,121 ----------------------------------------------------------------------------------------------------\n","2021-05-01 10:38:22,122 EPOCH 18 done: loss 6.4587 - lr 0.1000000\n","2021-05-01 10:38:28,765 DEV : loss 5.02623987197876 - score 0.7935\n","2021-05-01 10:38:28,921 BAD EPOCHS (no improvement): 0\n","saving best model\n","2021-05-01 10:38:31,410 ----------------------------------------------------------------------------------------------------\n","2021-05-01 10:38:47,411 epoch 19 - iter 63/635 - loss 6.35601944 - samples/sec: 252.19 - lr: 0.100000\n","2021-05-01 10:39:02,599 epoch 19 - iter 126/635 - loss 6.41037217 - samples/sec: 265.59 - lr: 0.100000\n","2021-05-01 10:39:17,843 epoch 19 - iter 189/635 - loss 6.41816486 - samples/sec: 264.62 - lr: 0.100000\n","2021-05-01 10:39:32,980 epoch 19 - iter 252/635 - loss 6.45103779 - samples/sec: 266.48 - lr: 0.100000\n","2021-05-01 10:39:48,021 epoch 19 - iter 315/635 - loss 6.43002831 - samples/sec: 268.19 - lr: 0.100000\n","2021-05-01 10:40:03,760 epoch 19 - iter 378/635 - loss 6.44875348 - samples/sec: 256.30 - lr: 0.100000\n","2021-05-01 10:40:20,079 epoch 19 - iter 441/635 - loss 6.46223157 - samples/sec: 247.17 - lr: 0.100000\n","2021-05-01 10:40:35,723 epoch 19 - iter 504/635 - loss 6.44294718 - samples/sec: 257.86 - lr: 0.100000\n","2021-05-01 10:40:52,307 epoch 19 - iter 567/635 - loss 6.46249333 - samples/sec: 243.24 - lr: 0.100000\n","2021-05-01 10:41:07,466 epoch 19 - iter 630/635 - loss 6.46185385 - samples/sec: 266.11 - lr: 0.100000\n","2021-05-01 10:41:08,530 ----------------------------------------------------------------------------------------------------\n","2021-05-01 10:41:08,531 EPOCH 19 done: loss 6.4646 - lr 0.1000000\n","2021-05-01 10:41:15,307 DEV : loss 4.989785194396973 - score 0.7952\n","2021-05-01 10:41:15,464 BAD EPOCHS (no improvement): 0\n","saving best model\n","2021-05-01 10:41:18,003 ----------------------------------------------------------------------------------------------------\n","2021-05-01 10:41:33,497 epoch 20 - iter 63/635 - loss 6.37028693 - samples/sec: 260.54 - lr: 0.100000\n","2021-05-01 10:41:48,884 epoch 20 - iter 126/635 - loss 6.44412295 - samples/sec: 262.14 - lr: 0.100000\n","2021-05-01 10:42:04,507 epoch 20 - iter 189/635 - loss 6.39939447 - samples/sec: 258.20 - lr: 0.100000\n","2021-05-01 10:42:20,187 epoch 20 - iter 252/635 - loss 6.42812613 - samples/sec: 257.33 - lr: 0.100000\n","2021-05-01 10:42:35,253 epoch 20 - iter 315/635 - loss 6.40041223 - samples/sec: 267.75 - lr: 0.100000\n","2021-05-01 10:42:50,511 epoch 20 - iter 378/635 - loss 6.39301693 - samples/sec: 264.39 - lr: 0.100000\n","2021-05-01 10:43:05,574 epoch 20 - iter 441/635 - loss 6.41253502 - samples/sec: 267.79 - lr: 0.100000\n","2021-05-01 10:43:21,124 epoch 20 - iter 504/635 - loss 6.39632354 - samples/sec: 259.41 - lr: 0.100000\n","2021-05-01 10:43:36,479 epoch 20 - iter 567/635 - loss 6.41526783 - samples/sec: 262.70 - lr: 0.100000\n","2021-05-01 10:43:51,986 epoch 20 - iter 630/635 - loss 6.42303124 - samples/sec: 260.14 - lr: 0.100000\n","2021-05-01 10:43:53,022 ----------------------------------------------------------------------------------------------------\n","2021-05-01 10:43:53,023 EPOCH 20 done: loss 6.4251 - lr 0.1000000\n","2021-05-01 10:44:01,876 DEV : loss 4.994434356689453 - score 0.7943\n","2021-05-01 10:44:02,035 BAD EPOCHS (no improvement): 1\n","2021-05-01 10:44:02,036 ----------------------------------------------------------------------------------------------------\n","2021-05-01 10:44:17,595 epoch 21 - iter 63/635 - loss 6.42438314 - samples/sec: 259.34 - lr: 0.100000\n","2021-05-01 10:44:33,206 epoch 21 - iter 126/635 - loss 6.39314774 - samples/sec: 258.41 - lr: 0.100000\n","2021-05-01 10:44:49,177 epoch 21 - iter 189/635 - loss 6.37178817 - samples/sec: 252.57 - lr: 0.100000\n","2021-05-01 10:45:04,160 epoch 21 - iter 252/635 - loss 6.34866030 - samples/sec: 269.22 - lr: 0.100000\n","2021-05-01 10:45:19,268 epoch 21 - iter 315/635 - loss 6.35940701 - samples/sec: 267.01 - lr: 0.100000\n","2021-05-01 10:45:34,391 epoch 21 - iter 378/635 - loss 6.41884683 - samples/sec: 266.73 - lr: 0.100000\n","2021-05-01 10:45:50,410 epoch 21 - iter 441/635 - loss 6.42135425 - samples/sec: 251.82 - lr: 0.100000\n","2021-05-01 10:46:05,874 epoch 21 - iter 504/635 - loss 6.43414676 - samples/sec: 260.88 - lr: 0.100000\n","2021-05-01 10:46:21,149 epoch 21 - iter 567/635 - loss 6.42369861 - samples/sec: 264.07 - lr: 0.100000\n","2021-05-01 10:46:36,278 epoch 21 - iter 630/635 - loss 6.40523226 - samples/sec: 266.63 - lr: 0.100000\n","2021-05-01 10:46:37,306 ----------------------------------------------------------------------------------------------------\n","2021-05-01 10:46:37,307 EPOCH 21 done: loss 6.4034 - lr 0.1000000\n","2021-05-01 10:46:43,961 DEV : loss 4.991807460784912 - score 0.7911\n","2021-05-01 10:46:44,116 BAD EPOCHS (no improvement): 2\n","2021-05-01 10:46:44,118 ----------------------------------------------------------------------------------------------------\n","2021-05-01 10:46:59,507 epoch 22 - iter 63/635 - loss 6.34989661 - samples/sec: 262.21 - lr: 0.100000\n","2021-05-01 10:47:14,556 epoch 22 - iter 126/635 - loss 6.32072991 - samples/sec: 268.05 - lr: 0.100000\n","2021-05-01 10:47:30,228 epoch 22 - iter 189/635 - loss 6.34303821 - samples/sec: 257.39 - lr: 0.100000\n","2021-05-01 10:47:45,510 epoch 22 - iter 252/635 - loss 6.35619980 - samples/sec: 263.96 - lr: 0.100000\n","2021-05-01 10:48:01,618 epoch 22 - iter 315/635 - loss 6.41007636 - samples/sec: 250.43 - lr: 0.100000\n","2021-05-01 10:48:17,129 epoch 22 - iter 378/635 - loss 6.40767628 - samples/sec: 260.09 - lr: 0.100000\n","2021-05-01 10:48:31,974 epoch 22 - iter 441/635 - loss 6.39029432 - samples/sec: 271.75 - lr: 0.100000\n","2021-05-01 10:48:47,300 epoch 22 - iter 504/635 - loss 6.37466748 - samples/sec: 263.20 - lr: 0.100000\n","2021-05-01 10:49:02,900 epoch 22 - iter 567/635 - loss 6.38637245 - samples/sec: 258.58 - lr: 0.100000\n","2021-05-01 10:49:18,329 epoch 22 - iter 630/635 - loss 6.37714043 - samples/sec: 261.44 - lr: 0.100000\n","2021-05-01 10:49:19,494 ----------------------------------------------------------------------------------------------------\n","2021-05-01 10:49:19,495 EPOCH 22 done: loss 6.3786 - lr 0.1000000\n","2021-05-01 10:49:26,193 DEV : loss 4.927341461181641 - score 0.7934\n","2021-05-01 10:49:26,349 BAD EPOCHS (no improvement): 3\n","2021-05-01 10:49:26,351 ----------------------------------------------------------------------------------------------------\n","2021-05-01 10:49:41,726 epoch 23 - iter 63/635 - loss 6.41943493 - samples/sec: 262.50 - lr: 0.100000\n","2021-05-01 10:49:57,597 epoch 23 - iter 126/635 - loss 6.44860041 - samples/sec: 254.15 - lr: 0.100000\n","2021-05-01 10:50:12,817 epoch 23 - iter 189/635 - loss 6.44031283 - samples/sec: 265.04 - lr: 0.100000\n","2021-05-01 10:50:28,308 epoch 23 - iter 252/635 - loss 6.40829029 - samples/sec: 260.39 - lr: 0.100000\n","2021-05-01 10:50:43,762 epoch 23 - iter 315/635 - loss 6.40047966 - samples/sec: 261.04 - lr: 0.100000\n","2021-05-01 10:50:59,126 epoch 23 - iter 378/635 - loss 6.37114754 - samples/sec: 262.56 - lr: 0.100000\n","2021-05-01 10:51:14,445 epoch 23 - iter 441/635 - loss 6.36717886 - samples/sec: 263.32 - lr: 0.100000\n","2021-05-01 10:51:30,297 epoch 23 - iter 504/635 - loss 6.36887095 - samples/sec: 254.49 - lr: 0.100000\n","2021-05-01 10:51:45,500 epoch 23 - iter 567/635 - loss 6.36700654 - samples/sec: 265.34 - lr: 0.100000\n","2021-05-01 10:52:01,200 epoch 23 - iter 630/635 - loss 6.36672055 - samples/sec: 256.93 - lr: 0.100000\n","2021-05-01 10:52:02,247 ----------------------------------------------------------------------------------------------------\n","2021-05-01 10:52:02,250 EPOCH 23 done: loss 6.3598 - lr 0.1000000\n","2021-05-01 10:52:08,938 DEV : loss 4.959728240966797 - score 0.793\n","Epoch    23: reducing learning rate of group 0 to 5.0000e-02.\n","2021-05-01 10:52:09,095 BAD EPOCHS (no improvement): 4\n","2021-05-01 10:52:09,096 ----------------------------------------------------------------------------------------------------\n","2021-05-01 10:52:24,423 epoch 24 - iter 63/635 - loss 6.27887647 - samples/sec: 263.33 - lr: 0.050000\n","2021-05-01 10:52:39,872 epoch 24 - iter 126/635 - loss 6.34430578 - samples/sec: 261.11 - lr: 0.050000\n","2021-05-01 10:52:55,356 epoch 24 - iter 189/635 - loss 6.35684780 - samples/sec: 260.50 - lr: 0.050000\n","2021-05-01 10:53:10,735 epoch 24 - iter 252/635 - loss 6.34141609 - samples/sec: 262.30 - lr: 0.050000\n","2021-05-01 10:53:26,774 epoch 24 - iter 315/635 - loss 6.40433425 - samples/sec: 251.50 - lr: 0.050000\n","2021-05-01 10:53:41,995 epoch 24 - iter 378/635 - loss 6.39992540 - samples/sec: 265.01 - lr: 0.050000\n","2021-05-01 10:53:57,132 epoch 24 - iter 441/635 - loss 6.36706100 - samples/sec: 266.48 - lr: 0.050000\n","2021-05-01 10:54:11,966 epoch 24 - iter 504/635 - loss 6.33817212 - samples/sec: 271.93 - lr: 0.050000\n","2021-05-01 10:54:27,113 epoch 24 - iter 567/635 - loss 6.33731364 - samples/sec: 266.32 - lr: 0.050000\n","2021-05-01 10:54:42,471 epoch 24 - iter 630/635 - loss 6.31624096 - samples/sec: 262.65 - lr: 0.050000\n","2021-05-01 10:54:43,596 ----------------------------------------------------------------------------------------------------\n","2021-05-01 10:54:43,597 EPOCH 24 done: loss 6.3153 - lr 0.0500000\n","2021-05-01 10:54:50,255 DEV : loss 4.936580181121826 - score 0.7957\n","2021-05-01 10:54:50,416 BAD EPOCHS (no improvement): 0\n","saving best model\n","2021-05-01 10:54:52,930 ----------------------------------------------------------------------------------------------------\n","2021-05-01 10:55:07,540 epoch 25 - iter 63/635 - loss 6.28007032 - samples/sec: 276.18 - lr: 0.050000\n","2021-05-01 10:55:22,706 epoch 25 - iter 126/635 - loss 6.28359970 - samples/sec: 266.02 - lr: 0.050000\n","2021-05-01 10:55:38,715 epoch 25 - iter 189/635 - loss 6.33361047 - samples/sec: 251.96 - lr: 0.050000\n","2021-05-01 10:55:54,143 epoch 25 - iter 252/635 - loss 6.24068344 - samples/sec: 261.46 - lr: 0.050000\n","2021-05-01 10:56:09,455 epoch 25 - iter 315/635 - loss 6.25132404 - samples/sec: 263.43 - lr: 0.050000\n","2021-05-01 10:56:24,649 epoch 25 - iter 378/635 - loss 6.23526979 - samples/sec: 265.49 - lr: 0.050000\n","2021-05-01 10:56:40,205 epoch 25 - iter 441/635 - loss 6.24943926 - samples/sec: 259.31 - lr: 0.050000\n","2021-05-01 10:56:55,982 epoch 25 - iter 504/635 - loss 6.25688947 - samples/sec: 255.69 - lr: 0.050000\n","2021-05-01 10:57:11,335 epoch 25 - iter 567/635 - loss 6.25921250 - samples/sec: 262.74 - lr: 0.050000\n","2021-05-01 10:57:26,890 epoch 25 - iter 630/635 - loss 6.27363411 - samples/sec: 259.37 - lr: 0.050000\n","2021-05-01 10:57:27,977 ----------------------------------------------------------------------------------------------------\n","2021-05-01 10:57:27,979 EPOCH 25 done: loss 6.2700 - lr 0.0500000\n","2021-05-01 10:57:34,602 DEV : loss 4.910365581512451 - score 0.7955\n","2021-05-01 10:57:34,760 BAD EPOCHS (no improvement): 1\n","2021-05-01 10:57:34,762 ----------------------------------------------------------------------------------------------------\n","2021-05-01 10:57:50,226 epoch 26 - iter 63/635 - loss 6.12873915 - samples/sec: 260.91 - lr: 0.050000\n","2021-05-01 10:58:05,599 epoch 26 - iter 126/635 - loss 6.21422136 - samples/sec: 262.39 - lr: 0.050000\n","2021-05-01 10:58:21,318 epoch 26 - iter 189/635 - loss 6.35024188 - samples/sec: 256.63 - lr: 0.050000\n","2021-05-01 10:58:36,692 epoch 26 - iter 252/635 - loss 6.37460351 - samples/sec: 262.37 - lr: 0.050000\n","2021-05-01 10:58:51,905 epoch 26 - iter 315/635 - loss 6.32543796 - samples/sec: 265.15 - lr: 0.050000\n","2021-05-01 10:59:07,790 epoch 26 - iter 378/635 - loss 6.37312124 - samples/sec: 253.94 - lr: 0.050000\n","2021-05-01 10:59:23,056 epoch 26 - iter 441/635 - loss 6.34267364 - samples/sec: 264.24 - lr: 0.050000\n","2021-05-01 10:59:38,675 epoch 26 - iter 504/635 - loss 6.31836976 - samples/sec: 258.24 - lr: 0.050000\n","2021-05-01 10:59:53,550 epoch 26 - iter 567/635 - loss 6.30718721 - samples/sec: 271.19 - lr: 0.050000\n","2021-05-01 11:00:08,821 epoch 26 - iter 630/635 - loss 6.29165388 - samples/sec: 264.17 - lr: 0.050000\n","2021-05-01 11:00:09,786 ----------------------------------------------------------------------------------------------------\n","2021-05-01 11:00:09,788 EPOCH 26 done: loss 6.2880 - lr 0.0500000\n","2021-05-01 11:00:16,406 DEV : loss 4.924932479858398 - score 0.7955\n","2021-05-01 11:00:16,569 BAD EPOCHS (no improvement): 2\n","2021-05-01 11:00:16,570 ----------------------------------------------------------------------------------------------------\n","2021-05-01 11:00:31,824 epoch 27 - iter 63/635 - loss 6.41725095 - samples/sec: 264.58 - lr: 0.050000\n","2021-05-01 11:00:47,656 epoch 27 - iter 126/635 - loss 6.39404274 - samples/sec: 254.78 - lr: 0.050000\n","2021-05-01 11:01:02,837 epoch 27 - iter 189/635 - loss 6.35007738 - samples/sec: 265.72 - lr: 0.050000\n","2021-05-01 11:01:18,098 epoch 27 - iter 252/635 - loss 6.34352289 - samples/sec: 264.31 - lr: 0.050000\n","2021-05-01 11:01:33,382 epoch 27 - iter 315/635 - loss 6.31697968 - samples/sec: 263.94 - lr: 0.050000\n","2021-05-01 11:01:48,716 epoch 27 - iter 378/635 - loss 6.30486854 - samples/sec: 263.06 - lr: 0.050000\n","2021-05-01 11:02:04,237 epoch 27 - iter 441/635 - loss 6.30261655 - samples/sec: 259.89 - lr: 0.050000\n","2021-05-01 11:02:19,412 epoch 27 - iter 504/635 - loss 6.28833699 - samples/sec: 265.83 - lr: 0.050000\n","2021-05-01 11:02:34,162 epoch 27 - iter 567/635 - loss 6.26939811 - samples/sec: 273.49 - lr: 0.050000\n","2021-05-01 11:02:49,819 epoch 27 - iter 630/635 - loss 6.26741784 - samples/sec: 257.64 - lr: 0.050000\n","2021-05-01 11:02:51,032 ----------------------------------------------------------------------------------------------------\n","2021-05-01 11:02:51,034 EPOCH 27 done: loss 6.2646 - lr 0.0500000\n","2021-05-01 11:02:57,650 DEV : loss 4.920487403869629 - score 0.7955\n","2021-05-01 11:02:57,806 BAD EPOCHS (no improvement): 3\n","2021-05-01 11:02:57,808 ----------------------------------------------------------------------------------------------------\n","2021-05-01 11:03:13,763 epoch 28 - iter 63/635 - loss 6.37393674 - samples/sec: 252.94 - lr: 0.050000\n","2021-05-01 11:03:28,646 epoch 28 - iter 126/635 - loss 6.21764481 - samples/sec: 271.03 - lr: 0.050000\n","2021-05-01 11:03:44,121 epoch 28 - iter 189/635 - loss 6.31300105 - samples/sec: 260.66 - lr: 0.050000\n","2021-05-01 11:03:59,250 epoch 28 - iter 252/635 - loss 6.30954745 - samples/sec: 266.63 - lr: 0.050000\n","2021-05-01 11:04:14,470 epoch 28 - iter 315/635 - loss 6.28663222 - samples/sec: 265.04 - lr: 0.050000\n","2021-05-01 11:04:29,873 epoch 28 - iter 378/635 - loss 6.28437442 - samples/sec: 261.87 - lr: 0.050000\n","2021-05-01 11:04:45,161 epoch 28 - iter 441/635 - loss 6.29338149 - samples/sec: 263.87 - lr: 0.050000\n","2021-05-01 11:05:02,517 epoch 28 - iter 504/635 - loss 6.30503538 - samples/sec: 232.40 - lr: 0.050000\n","2021-05-01 11:05:18,043 epoch 28 - iter 567/635 - loss 6.29632414 - samples/sec: 259.81 - lr: 0.050000\n","2021-05-01 11:05:33,085 epoch 28 - iter 630/635 - loss 6.27202777 - samples/sec: 268.17 - lr: 0.050000\n","2021-05-01 11:05:34,068 ----------------------------------------------------------------------------------------------------\n","2021-05-01 11:05:34,070 EPOCH 28 done: loss 6.2652 - lr 0.0500000\n","2021-05-01 11:05:40,707 DEV : loss 4.901998519897461 - score 0.7959\n","2021-05-01 11:05:40,863 BAD EPOCHS (no improvement): 0\n","saving best model\n","2021-05-01 11:05:43,378 ----------------------------------------------------------------------------------------------------\n","2021-05-01 11:05:58,280 epoch 29 - iter 63/635 - loss 6.15860039 - samples/sec: 270.83 - lr: 0.050000\n","2021-05-01 11:06:13,956 epoch 29 - iter 126/635 - loss 6.25396917 - samples/sec: 257.33 - lr: 0.050000\n","2021-05-01 11:06:29,759 epoch 29 - iter 189/635 - loss 6.29054735 - samples/sec: 255.27 - lr: 0.050000\n","2021-05-01 11:06:45,554 epoch 29 - iter 252/635 - loss 6.28242232 - samples/sec: 255.38 - lr: 0.050000\n","2021-05-01 11:07:00,622 epoch 29 - iter 315/635 - loss 6.26875989 - samples/sec: 267.71 - lr: 0.050000\n","2021-05-01 11:07:16,418 epoch 29 - iter 378/635 - loss 6.26207167 - samples/sec: 255.36 - lr: 0.050000\n","2021-05-01 11:07:31,616 epoch 29 - iter 441/635 - loss 6.24082525 - samples/sec: 265.43 - lr: 0.050000\n","2021-05-01 11:07:46,885 epoch 29 - iter 504/635 - loss 6.25054458 - samples/sec: 264.19 - lr: 0.050000\n","2021-05-01 11:08:02,304 epoch 29 - iter 567/635 - loss 6.25115722 - samples/sec: 261.60 - lr: 0.050000\n","2021-05-01 11:08:17,424 epoch 29 - iter 630/635 - loss 6.23680396 - samples/sec: 266.80 - lr: 0.050000\n","2021-05-01 11:08:18,625 ----------------------------------------------------------------------------------------------------\n","2021-05-01 11:08:18,626 EPOCH 29 done: loss 6.2450 - lr 0.0500000\n","2021-05-01 11:08:25,229 DEV : loss 4.884469509124756 - score 0.7961\n","2021-05-01 11:08:25,385 BAD EPOCHS (no improvement): 0\n","saving best model\n","2021-05-01 11:08:27,884 ----------------------------------------------------------------------------------------------------\n","2021-05-01 11:08:43,083 epoch 30 - iter 63/635 - loss 6.02414188 - samples/sec: 265.57 - lr: 0.050000\n","2021-05-01 11:08:58,636 epoch 30 - iter 126/635 - loss 6.20158135 - samples/sec: 259.35 - lr: 0.050000\n","2021-05-01 11:09:14,111 epoch 30 - iter 189/635 - loss 6.28846838 - samples/sec: 260.67 - lr: 0.050000\n","2021-05-01 11:09:29,524 epoch 30 - iter 252/635 - loss 6.28758190 - samples/sec: 261.71 - lr: 0.050000\n","2021-05-01 11:09:44,841 epoch 30 - iter 315/635 - loss 6.26563515 - samples/sec: 263.35 - lr: 0.050000\n","2021-05-01 11:10:00,154 epoch 30 - iter 378/635 - loss 6.25417992 - samples/sec: 263.41 - lr: 0.050000\n","2021-05-01 11:10:15,223 epoch 30 - iter 441/635 - loss 6.24465964 - samples/sec: 267.70 - lr: 0.050000\n","2021-05-01 11:10:30,328 epoch 30 - iter 504/635 - loss 6.22326431 - samples/sec: 267.04 - lr: 0.050000\n","2021-05-01 11:10:45,591 epoch 30 - iter 567/635 - loss 6.22423295 - samples/sec: 264.29 - lr: 0.050000\n","2021-05-01 11:11:00,787 epoch 30 - iter 630/635 - loss 6.22857451 - samples/sec: 265.47 - lr: 0.050000\n","2021-05-01 11:11:02,031 ----------------------------------------------------------------------------------------------------\n","2021-05-01 11:11:02,032 EPOCH 30 done: loss 6.2355 - lr 0.0500000\n","2021-05-01 11:11:08,635 DEV : loss 4.871624946594238 - score 0.7974\n","2021-05-01 11:11:08,790 BAD EPOCHS (no improvement): 0\n","saving best model\n","2021-05-01 11:11:13,720 ----------------------------------------------------------------------------------------------------\n","2021-05-01 11:11:13,727 Testing using best model ...\n","2021-05-01 11:11:13,730 loading file resources/taggers/example-ner/best-model.pt\n","2021-05-01 11:11:20,062 0.8509\t0.7591\t0.8024\n","2021-05-01 11:11:20,064 \n","Results:\n","- F1-score (micro) 0.8024\n","- F1-score (macro) 0.6017\n","\n","By class:\n",".          tp: 3631 - fp: 0 - fn: 0 - precision: 1.0000 - recall: 1.0000 - f1-score: 1.0000\n","art        tp: 0 - fp: 0 - fn: 47 - precision: 0.0000 - recall: 0.0000 - f1-score: 0.0000\n","eve        tp: 8 - fp: 3 - fn: 34 - precision: 0.7273 - recall: 0.1905 - f1-score: 0.3019\n","geo        tp: 2702 - fp: 860 - fn: 606 - precision: 0.7586 - recall: 0.8168 - f1-score: 0.7866\n","gpe        tp: 1051 - fp: 104 - fn: 130 - precision: 0.9100 - recall: 0.8899 - f1-score: 0.8998\n","nat        tp: 7 - fp: 5 - fn: 14 - precision: 0.5833 - recall: 0.3333 - f1-score: 0.4242\n","org        tp: 777 - fp: 286 - fn: 1804 - precision: 0.7310 - recall: 0.3010 - f1-score: 0.4265\n","per        tp: 1964 - fp: 573 - fn: 443 - precision: 0.7741 - recall: 0.8160 - f1-score: 0.7945\n","tim        tp: 1316 - fp: 177 - fn: 557 - precision: 0.8814 - recall: 0.7026 - f1-score: 0.7819\n","2021-05-01 11:11:20,071 ----------------------------------------------------------------------------------------------------\n"],"name":"stdout"},{"output_type":"execute_result","data":{"text/plain":["{'dev_loss_history': [7.366448879241943,\n","  6.434582710266113,\n","  5.904755115509033,\n","  5.7200212478637695,\n","  5.672123908996582,\n","  5.504039287567139,\n","  5.449540615081787,\n","  5.392806053161621,\n","  5.37333345413208,\n","  5.257050514221191,\n","  5.211832046508789,\n","  5.187159538269043,\n","  5.126097202301025,\n","  5.1455302238464355,\n","  5.105213165283203,\n","  5.104139804840088,\n","  5.07221794128418,\n","  5.02623987197876,\n","  4.989785194396973,\n","  4.994434356689453,\n","  4.991807460784912,\n","  4.927341461181641,\n","  4.959728240966797,\n","  4.936580181121826,\n","  4.910365581512451,\n","  4.924932479858398,\n","  4.920487403869629,\n","  4.901998519897461,\n","  4.884469509124756,\n","  4.871624946594238],\n"," 'dev_score_history': [0.7271155869532543,\n","  0.7566694800190817,\n","  0.7692031972472132,\n","  0.7736233854520733,\n","  0.7746778076759667,\n","  0.7762205008106646,\n","  0.7813090779382914,\n","  0.7829100940793662,\n","  0.7844533029612757,\n","  0.7893399223438051,\n","  0.7894400227401933,\n","  0.7907108467856131,\n","  0.7909500986749366,\n","  0.7877782874078255,\n","  0.7912803453889165,\n","  0.790830129574256,\n","  0.7909291249022534,\n","  0.7934767425618103,\n","  0.7951924751785403,\n","  0.794330202841881,\n","  0.7910979437801997,\n","  0.7933930571108623,\n","  0.7929949664429531,\n","  0.7957207285949026,\n","  0.7955315063710944,\n","  0.7954767555493508,\n","  0.795523269959334,\n","  0.795874125874126,\n","  0.7960826682466107,\n","  0.7973719158453904],\n"," 'test_score': 0.8023813692873403,\n"," 'train_loss_history': [11.879263609037624,\n","  8.619394586217686,\n","  7.922587698087917,\n","  7.581665310897226,\n","  7.345881294640969,\n","  7.1829400738393225,\n","  7.06127489795835,\n","  6.961241409722276,\n","  6.842466121583473,\n","  6.7848152573653095,\n","  6.72220410775012,\n","  6.673371574446911,\n","  6.634478655387097,\n","  6.582150185952975,\n","  6.544912804581049,\n","  6.5210225184132735,\n","  6.464603620063602,\n","  6.458664288483266,\n","  6.464569172145813,\n","  6.425115074698381,\n","  6.403401096223846,\n","  6.378561255687804,\n","  6.359812439145066,\n","  6.315345434128769,\n","  6.269972614228256,\n","  6.287962256454107,\n","  6.264570313551294,\n","  6.265177082452248,\n","  6.2450055148657855,\n","  6.235495721261333]}"]},"metadata":{"tags":[]},"execution_count":13}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"l1gysd-ixzqW","executionInfo":{"elapsed":1589,"status":"ok","timestamp":1619867674591,"user":{"displayName":"aftab anjum","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gjr9EAQIKikAsQFruoHrEH12B72Qi-qbRgbTxCSNg=s64","userId":"09823110291100232267"},"user_tz":-300},"outputId":"eb86a75b-a991-4d8b-96b9-1e07eb947851"},"source":["model = SequenceTagger.load('resources/taggers/example-ner/final-model.pt')\n","from flair.data import Sentence\n","# create example sentence\n","sentence = Sentence('i am John from lahore')\n","# predict tags and print\n","model.predict(sentence)\n","print(sentence.to_tagged_string())\n","#print(sentence.labels)\n","print(sentence)\n","# iterate over entities and print\n","for entity in sentence.get_spans('ner'):\n","    print(entity)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["2021-05-01 11:14:33,511 loading file resources/taggers/example-ner/final-model.pt\n","i am John <B-per> from lahore <B-geo>\n","Sentence: \"i am John from lahore\"   [− Tokens: 5  − Token-Labels: \"i am John <B-per> from lahore <B-geo>\"]\n","Span [3]: \"John\"   [− Labels: per (0.6409)]\n","Span [5]: \"lahore\"   [− Labels: geo (0.7981)]\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"RT1p14Pboh7v","executionInfo":{"elapsed":2114,"status":"ok","timestamp":1619868430579,"user":{"displayName":"aftab anjum","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gjr9EAQIKikAsQFruoHrEH12B72Qi-qbRgbTxCSNg=s64","userId":"09823110291100232267"},"user_tz":-300},"outputId":"78a4b1a7-bdb2-42d5-9679-f7cf8912849f"},"source":["model = SequenceTagger.load('resources/taggers/example-ner/final-model.pt')\n","from flair.data import Sentence\n","# create example sentence\n","sentence = Sentence('The first episode aired on 19 November 2004. The show is hosted by producer and director Karan Johar and is produced by SOL India. The show wrapped up its sixth season on 17 March 2019')\n","# predict tags and print\n","model.predict(sentence)\n","print(sentence.to_tagged_string())\n","#print(sentence.labels)\n","print(sentence)\n","# iterate over entities and print\n","for entity in sentence.get_spans('ner'):\n","    print(entity)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["2021-05-01 11:27:08,961 loading file resources/taggers/example-ner/final-model.pt\n","The first episode aired on 19 November <B-tim> 2004 <B-tim> . The show is hosted by producer and director Karan <B-per> Johar <B-per> and is produced by SOL <B-per> India <B-geo> . The show wrapped up its sixth season on 17 March <B-tim> 2019 <.>\n","Sentence: \"The first episode aired on 19 November 2004 . The show is hosted by producer and director Karan Johar and is produced by SOL India . The show wrapped up its sixth season on 17 March 2019\"   [− Tokens: 37  − Token-Labels: \"The first episode aired on 19 November <B-tim> 2004 <B-tim> . The show is hosted by producer and director Karan <B-per> Johar <B-per> and is produced by SOL <B-per> India <B-geo> . The show wrapped up its sixth season on 17 March <B-tim> 2019 <.>\"]\n","Span [7]: \"November\"   [− Labels: tim (0.8937)]\n","Span [8]: \"2004\"   [− Labels: tim (0.5516)]\n","Span [18]: \"Karan\"   [− Labels: per (0.6169)]\n","Span [19]: \"Johar\"   [− Labels: per (0.2906)]\n","Span [24]: \"SOL\"   [− Labels: per (0.3067)]\n","Span [25]: \"India\"   [− Labels: geo (0.8917)]\n","Span [36]: \"March\"   [− Labels: tim (0.9001)]\n","Span [37]: \"2019\"   [− Labels: . (0.6901)]\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"xllxYO-LsF9c"},"source":["## Flair NER model Stacked Embeddings (Glove, forward and backward Flair embeddings)\n","\n","\n"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"84m9IM8spa8i","executionInfo":{"status":"ok","timestamp":1619966168041,"user_tz":-300,"elapsed":3264829,"user":{"displayName":"AI Team","photoUrl":"","userId":"17689037188246991143"}},"outputId":"b0949d5b-1b43-49e5-82b3-c3ffbade0d51"},"source":["from flair.embeddings import WordEmbeddings, FlairEmbeddings\n","\n","tag_type = 'ner'\n","\n","tag_dictionary = corpus.make_tag_dictionary(tag_type=tag_type)\n","\n","# init standard GloVe embedding\n","#glove_embedding = WordEmbeddings('glove')\n","\n","# init Flair forward and backwards embeddings\n","flair_embedding_forward = FlairEmbeddings('news-forward')\n","flair_embedding_backward = FlairEmbeddings('news-backward')\n","\n","from flair.embeddings import StackedEmbeddings\n","\n","# create a StackedEmbedding object that combines glove and forward/backward flair embeddings\n","stacked_embeddings = StackedEmbeddings([\n","            \n","                                        flair_embedding_forward,\n","                                        flair_embedding_backward,\n","                                       ])\n","\n","\n","from flair.models import SequenceTagger\n","tagger: SequenceTagger = SequenceTagger(hidden_size=256,\n","                                        embeddings=stacked_embeddings,\n","                                        tag_dictionary=tag_dictionary,\n","                                        tag_type=tag_type,\n","                                        use_crf=True)\n","# 5. create the text classifier\n","# classifier = TextClassifier(document_embeddings, label_dictionary=label_dict)\n","from flair.trainers import ModelTrainer\n","\n","trainer: ModelTrainer = ModelTrainer(tagger, corpus)\n","\n","# 6. initialize the text classifier trainer\n","# trainer = ModelTrainer(classifier, corpus)\n","history=trainer.train('/content/gdrive/MyDrive/NLP projects/NER projects/project 1/',\n","              learning_rate=0.1,\n","              mini_batch_size=15,\n","              max_epochs=20)\n"],"execution_count":19,"outputs":[{"output_type":"stream","text":["2021-05-02 11:54:30,865 https://flair.informatik.hu-berlin.de/resources/embeddings/flair/news-forward-0.4.1.pt not found in cache, downloading to /tmp/tmpapuxg7ql\n"],"name":"stdout"},{"output_type":"stream","text":["100%|██████████| 73034624/73034624 [00:03<00:00, 18527758.37B/s]"],"name":"stderr"},{"output_type":"stream","text":["2021-05-02 11:54:35,179 copying /tmp/tmpapuxg7ql to cache at /root/.flair/embeddings/news-forward-0.4.1.pt\n"],"name":"stdout"},{"output_type":"stream","text":["\n"],"name":"stderr"},{"output_type":"stream","text":["2021-05-02 11:54:35,244 removing temp file /tmp/tmpapuxg7ql\n","2021-05-02 11:54:38,907 https://flair.informatik.hu-berlin.de/resources/embeddings/flair/news-backward-0.4.1.pt not found in cache, downloading to /tmp/tmpfi5vdjta\n"],"name":"stdout"},{"output_type":"stream","text":["100%|██████████| 73034575/73034575 [00:04<00:00, 17441121.34B/s]"],"name":"stderr"},{"output_type":"stream","text":["2021-05-02 11:54:43,463 copying /tmp/tmpfi5vdjta to cache at /root/.flair/embeddings/news-backward-0.4.1.pt\n"],"name":"stdout"},{"output_type":"stream","text":["\n"],"name":"stderr"},{"output_type":"stream","text":["2021-05-02 11:54:43,527 removing temp file /tmp/tmpfi5vdjta\n","2021-05-02 11:54:44,834 ----------------------------------------------------------------------------------------------------\n","2021-05-02 11:54:44,836 Model: \"SequenceTagger(\n","  (embeddings): StackedEmbeddings(\n","    (list_embedding_0): FlairEmbeddings(\n","      (lm): LanguageModel(\n","        (drop): Dropout(p=0.05, inplace=False)\n","        (encoder): Embedding(300, 100)\n","        (rnn): LSTM(100, 2048)\n","        (decoder): Linear(in_features=2048, out_features=300, bias=True)\n","      )\n","    )\n","    (list_embedding_1): FlairEmbeddings(\n","      (lm): LanguageModel(\n","        (drop): Dropout(p=0.05, inplace=False)\n","        (encoder): Embedding(300, 100)\n","        (rnn): LSTM(100, 2048)\n","        (decoder): Linear(in_features=2048, out_features=300, bias=True)\n","      )\n","    )\n","  )\n","  (word_dropout): WordDropout(p=0.05)\n","  (locked_dropout): LockedDropout(p=0.5)\n","  (embedding2nn): Linear(in_features=4096, out_features=4096, bias=True)\n","  (rnn): LSTM(4096, 256, batch_first=True, bidirectional=True)\n","  (linear): Linear(in_features=512, out_features=23, bias=True)\n","  (beta): 1.0\n","  (weights): None\n","  (weight_tensor) None\n",")\"\n","2021-05-02 11:54:44,838 ----------------------------------------------------------------------------------------------------\n","2021-05-02 11:54:44,841 Corpus: \"Corpus: 40586 train + 3546 dev + 3632 test sentences\"\n","2021-05-02 11:54:44,842 ----------------------------------------------------------------------------------------------------\n","2021-05-02 11:54:44,844 Parameters:\n","2021-05-02 11:54:44,845  - learning_rate: \"0.1\"\n","2021-05-02 11:54:44,847  - mini_batch_size: \"15\"\n","2021-05-02 11:54:44,849  - patience: \"3\"\n","2021-05-02 11:54:44,851  - anneal_factor: \"0.5\"\n","2021-05-02 11:54:44,853  - max_epochs: \"20\"\n","2021-05-02 11:54:44,854  - shuffle: \"True\"\n","2021-05-02 11:54:44,856  - train_with_dev: \"False\"\n","2021-05-02 11:54:44,857  - batch_growth_annealing: \"False\"\n","2021-05-02 11:54:44,859 ----------------------------------------------------------------------------------------------------\n","2021-05-02 11:54:44,865 Model training base path: \"/content/gdrive/MyDrive/NLP projects/NER projects/project 1\"\n","2021-05-02 11:54:44,867 ----------------------------------------------------------------------------------------------------\n","2021-05-02 11:54:44,869 Device: cuda:0\n","2021-05-02 11:54:44,873 ----------------------------------------------------------------------------------------------------\n","2021-05-02 11:54:44,875 Embeddings storage mode: cpu\n","2021-05-02 11:54:44,883 ----------------------------------------------------------------------------------------------------\n","2021-05-02 11:57:24,709 epoch 1 - iter 270/2706 - loss 10.34777189 - samples/sec: 25.34 - lr: 0.100000\n","2021-05-02 12:00:12,458 epoch 1 - iter 540/2706 - loss 8.45390791 - samples/sec: 24.15 - lr: 0.100000\n","2021-05-02 12:02:51,059 epoch 1 - iter 810/2706 - loss 7.69470282 - samples/sec: 25.54 - lr: 0.100000\n","2021-05-02 12:05:36,095 epoch 1 - iter 1080/2706 - loss 7.25681913 - samples/sec: 24.54 - lr: 0.100000\n","2021-05-02 12:08:20,039 epoch 1 - iter 1350/2706 - loss 6.94201037 - samples/sec: 24.71 - lr: 0.100000\n","2021-05-02 12:11:01,423 epoch 1 - iter 1620/2706 - loss 6.72191215 - samples/sec: 25.10 - lr: 0.100000\n","2021-05-02 12:13:42,699 epoch 1 - iter 1890/2706 - loss 6.55123595 - samples/sec: 25.11 - lr: 0.100000\n","2021-05-02 12:16:25,784 epoch 1 - iter 2160/2706 - loss 6.39640135 - samples/sec: 24.84 - lr: 0.100000\n","2021-05-02 12:19:09,266 epoch 1 - iter 2430/2706 - loss 6.28473968 - samples/sec: 24.78 - lr: 0.100000\n","2021-05-02 12:21:42,224 epoch 1 - iter 2700/2706 - loss 6.16856299 - samples/sec: 26.48 - lr: 0.100000\n","2021-05-02 12:21:45,447 ----------------------------------------------------------------------------------------------------\n","2021-05-02 12:21:45,449 EPOCH 1 done: loss 6.1659 - lr 0.1000000\n","2021-05-02 12:23:45,385 DEV : loss 4.51550817489624 - score 0.8031\n","2021-05-02 12:23:45,625 BAD EPOCHS (no improvement): 0\n","saving best model\n","2021-05-02 12:23:46,704 ----------------------------------------------------------------------------------------------------\n","2021-05-02 12:24:27,093 epoch 2 - iter 270/2706 - loss 5.15313969 - samples/sec: 100.31 - lr: 0.100000\n","2021-05-02 12:25:06,999 epoch 2 - iter 540/2706 - loss 5.16485280 - samples/sec: 101.52 - lr: 0.100000\n","2021-05-02 12:25:46,360 epoch 2 - iter 810/2706 - loss 5.16763546 - samples/sec: 102.93 - lr: 0.100000\n","2021-05-02 12:26:24,815 epoch 2 - iter 1080/2706 - loss 5.09626412 - samples/sec: 105.35 - lr: 0.100000\n","2021-05-02 12:27:04,046 epoch 2 - iter 1350/2706 - loss 5.08073868 - samples/sec: 103.27 - lr: 0.100000\n","2021-05-02 12:27:45,001 epoch 2 - iter 1620/2706 - loss 5.09813302 - samples/sec: 98.92 - lr: 0.100000\n","2021-05-02 12:28:25,140 epoch 2 - iter 1890/2706 - loss 5.07707975 - samples/sec: 100.93 - lr: 0.100000\n","2021-05-02 12:29:04,005 epoch 2 - iter 2160/2706 - loss 5.05943947 - samples/sec: 104.24 - lr: 0.100000\n","2021-05-02 12:29:43,331 epoch 2 - iter 2430/2706 - loss 5.04975032 - samples/sec: 103.02 - lr: 0.100000\n","2021-05-02 12:30:24,368 epoch 2 - iter 2700/2706 - loss 5.05157442 - samples/sec: 98.72 - lr: 0.100000\n","2021-05-02 12:30:25,262 ----------------------------------------------------------------------------------------------------\n","2021-05-02 12:30:25,264 EPOCH 2 done: loss 5.0510 - lr 0.1000000\n","2021-05-02 12:30:38,505 DEV : loss 4.282794952392578 - score 0.8073\n","2021-05-02 12:30:38,746 BAD EPOCHS (no improvement): 0\n","saving best model\n","2021-05-02 12:30:40,047 ----------------------------------------------------------------------------------------------------\n","2021-05-02 12:31:20,088 epoch 3 - iter 270/2706 - loss 4.83979247 - samples/sec: 101.19 - lr: 0.100000\n","2021-05-02 12:32:00,082 epoch 3 - iter 540/2706 - loss 4.87352306 - samples/sec: 101.30 - lr: 0.100000\n","2021-05-02 12:32:40,236 epoch 3 - iter 810/2706 - loss 4.85744474 - samples/sec: 100.89 - lr: 0.100000\n","2021-05-02 12:33:20,218 epoch 3 - iter 1080/2706 - loss 4.86666004 - samples/sec: 101.33 - lr: 0.100000\n","2021-05-02 12:34:00,995 epoch 3 - iter 1350/2706 - loss 4.87249165 - samples/sec: 99.35 - lr: 0.100000\n","2021-05-02 12:34:41,233 epoch 3 - iter 1620/2706 - loss 4.85231964 - samples/sec: 100.68 - lr: 0.100000\n","2021-05-02 12:35:21,473 epoch 3 - iter 1890/2706 - loss 4.84773776 - samples/sec: 100.68 - lr: 0.100000\n","2021-05-02 12:36:01,503 epoch 3 - iter 2160/2706 - loss 4.84201565 - samples/sec: 101.20 - lr: 0.100000\n","2021-05-02 12:36:41,517 epoch 3 - iter 2430/2706 - loss 4.86182475 - samples/sec: 101.25 - lr: 0.100000\n","2021-05-02 12:37:21,296 epoch 3 - iter 2700/2706 - loss 4.85415634 - samples/sec: 101.85 - lr: 0.100000\n","2021-05-02 12:37:22,201 ----------------------------------------------------------------------------------------------------\n","2021-05-02 12:37:22,203 EPOCH 3 done: loss 4.8538 - lr 0.1000000\n","2021-05-02 12:37:35,611 DEV : loss 4.162961483001709 - score 0.8111\n","2021-05-02 12:37:35,851 BAD EPOCHS (no improvement): 0\n","saving best model\n","2021-05-02 12:37:37,122 ----------------------------------------------------------------------------------------------------\n","2021-05-02 12:38:16,869 epoch 4 - iter 270/2706 - loss 4.81416912 - samples/sec: 101.95 - lr: 0.100000\n","2021-05-02 12:38:57,536 epoch 4 - iter 540/2706 - loss 4.80975842 - samples/sec: 99.62 - lr: 0.100000\n","2021-05-02 12:39:37,804 epoch 4 - iter 810/2706 - loss 4.77210798 - samples/sec: 100.60 - lr: 0.100000\n","2021-05-02 12:40:17,256 epoch 4 - iter 1080/2706 - loss 4.76117985 - samples/sec: 102.69 - lr: 0.100000\n","2021-05-02 12:40:59,011 epoch 4 - iter 1350/2706 - loss 4.78018187 - samples/sec: 97.03 - lr: 0.100000\n","2021-05-02 12:41:39,528 epoch 4 - iter 1620/2706 - loss 4.78244516 - samples/sec: 99.99 - lr: 0.100000\n","2021-05-02 12:42:18,210 epoch 4 - iter 1890/2706 - loss 4.75908771 - samples/sec: 104.73 - lr: 0.100000\n","2021-05-02 12:42:57,501 epoch 4 - iter 2160/2706 - loss 4.75819322 - samples/sec: 103.11 - lr: 0.100000\n","2021-05-02 12:43:38,309 epoch 4 - iter 2430/2706 - loss 4.76773756 - samples/sec: 99.27 - lr: 0.100000\n","2021-05-02 12:44:17,336 epoch 4 - iter 2700/2706 - loss 4.74504158 - samples/sec: 103.81 - lr: 0.100000\n","2021-05-02 12:44:18,295 ----------------------------------------------------------------------------------------------------\n","2021-05-02 12:44:18,297 EPOCH 4 done: loss 4.7446 - lr 0.1000000\n","2021-05-02 12:44:34,097 DEV : loss 4.118988513946533 - score 0.812\n","2021-05-02 12:44:34,343 BAD EPOCHS (no improvement): 0\n","saving best model\n","2021-05-02 12:44:35,390 ----------------------------------------------------------------------------------------------------\n","2021-05-02 12:45:17,087 epoch 5 - iter 270/2706 - loss 4.62724946 - samples/sec: 97.16 - lr: 0.100000\n","2021-05-02 12:45:56,335 epoch 5 - iter 540/2706 - loss 4.58643517 - samples/sec: 103.22 - lr: 0.100000\n","2021-05-02 12:46:36,733 epoch 5 - iter 810/2706 - loss 4.65701728 - samples/sec: 100.28 - lr: 0.100000\n","2021-05-02 12:47:17,729 epoch 5 - iter 1080/2706 - loss 4.67914883 - samples/sec: 98.82 - lr: 0.100000\n","2021-05-02 12:47:58,270 epoch 5 - iter 1350/2706 - loss 4.67678514 - samples/sec: 99.93 - lr: 0.100000\n","2021-05-02 12:48:38,008 epoch 5 - iter 1620/2706 - loss 4.66177898 - samples/sec: 101.95 - lr: 0.100000\n","2021-05-02 12:49:17,351 epoch 5 - iter 1890/2706 - loss 4.65122423 - samples/sec: 102.97 - lr: 0.100000\n","2021-05-02 12:49:56,771 epoch 5 - iter 2160/2706 - loss 4.63253264 - samples/sec: 102.77 - lr: 0.100000\n","2021-05-02 12:50:36,889 epoch 5 - iter 2430/2706 - loss 4.62792384 - samples/sec: 100.98 - lr: 0.100000\n","2021-05-02 12:51:16,975 epoch 5 - iter 2700/2706 - loss 4.63819800 - samples/sec: 101.06 - lr: 0.100000\n","2021-05-02 12:51:17,930 ----------------------------------------------------------------------------------------------------\n","2021-05-02 12:51:17,932 EPOCH 5 done: loss 4.6408 - lr 0.1000000\n","2021-05-02 12:51:31,296 DEV : loss 4.060299873352051 - score 0.8102\n","2021-05-02 12:51:31,538 BAD EPOCHS (no improvement): 1\n","2021-05-02 12:51:31,543 ----------------------------------------------------------------------------------------------------\n","2021-05-02 12:52:12,270 epoch 6 - iter 270/2706 - loss 4.73738374 - samples/sec: 99.48 - lr: 0.100000\n","2021-05-02 12:52:53,523 epoch 6 - iter 540/2706 - loss 4.66916912 - samples/sec: 98.20 - lr: 0.100000\n","2021-05-02 12:53:33,200 epoch 6 - iter 810/2706 - loss 4.59486077 - samples/sec: 102.11 - lr: 0.100000\n","2021-05-02 12:54:12,999 epoch 6 - iter 1080/2706 - loss 4.56190108 - samples/sec: 101.79 - lr: 0.100000\n","2021-05-02 12:54:52,913 epoch 6 - iter 1350/2706 - loss 4.57007498 - samples/sec: 101.50 - lr: 0.100000\n","2021-05-02 12:55:32,196 epoch 6 - iter 1620/2706 - loss 4.57097741 - samples/sec: 103.13 - lr: 0.100000\n","2021-05-02 12:56:11,014 epoch 6 - iter 1890/2706 - loss 4.55947931 - samples/sec: 104.37 - lr: 0.100000\n","2021-05-02 12:56:51,168 epoch 6 - iter 2160/2706 - loss 4.56472318 - samples/sec: 100.89 - lr: 0.100000\n","2021-05-02 12:57:30,813 epoch 6 - iter 2430/2706 - loss 4.55972412 - samples/sec: 102.19 - lr: 0.100000\n","2021-05-02 12:58:10,421 epoch 6 - iter 2700/2706 - loss 4.56423035 - samples/sec: 102.28 - lr: 0.100000\n","2021-05-02 12:58:11,338 ----------------------------------------------------------------------------------------------------\n","2021-05-02 12:58:11,340 EPOCH 6 done: loss 4.5632 - lr 0.1000000\n","2021-05-02 12:58:24,629 DEV : loss 4.047110080718994 - score 0.8132\n","2021-05-02 12:58:24,866 BAD EPOCHS (no improvement): 0\n","saving best model\n","2021-05-02 12:58:25,949 ----------------------------------------------------------------------------------------------------\n","2021-05-02 12:59:07,198 epoch 7 - iter 270/2706 - loss 4.59787320 - samples/sec: 98.22 - lr: 0.100000\n","2021-05-02 12:59:48,511 epoch 7 - iter 540/2706 - loss 4.60202947 - samples/sec: 98.06 - lr: 0.100000\n","2021-05-02 13:00:28,649 epoch 7 - iter 810/2706 - loss 4.56093818 - samples/sec: 100.93 - lr: 0.100000\n","2021-05-02 13:01:08,431 epoch 7 - iter 1080/2706 - loss 4.52489992 - samples/sec: 101.84 - lr: 0.100000\n","2021-05-02 13:01:48,235 epoch 7 - iter 1350/2706 - loss 4.52633839 - samples/sec: 101.78 - lr: 0.100000\n","2021-05-02 13:02:27,760 epoch 7 - iter 1620/2706 - loss 4.51599024 - samples/sec: 102.50 - lr: 0.100000\n","2021-05-02 13:03:06,434 epoch 7 - iter 1890/2706 - loss 4.49198580 - samples/sec: 104.75 - lr: 0.100000\n","2021-05-02 13:03:44,722 epoch 7 - iter 2160/2706 - loss 4.48837306 - samples/sec: 105.81 - lr: 0.100000\n","2021-05-02 13:04:23,425 epoch 7 - iter 2430/2706 - loss 4.49895693 - samples/sec: 104.67 - lr: 0.100000\n","2021-05-02 13:05:04,114 epoch 7 - iter 2700/2706 - loss 4.52291023 - samples/sec: 99.57 - lr: 0.100000\n","2021-05-02 13:05:05,027 ----------------------------------------------------------------------------------------------------\n","2021-05-02 13:05:05,029 EPOCH 7 done: loss 4.5232 - lr 0.1000000\n","2021-05-02 13:05:18,297 DEV : loss 4.056057453155518 - score 0.8128\n","2021-05-02 13:05:18,548 BAD EPOCHS (no improvement): 1\n","2021-05-02 13:05:18,552 ----------------------------------------------------------------------------------------------------\n","2021-05-02 13:05:58,709 epoch 8 - iter 270/2706 - loss 4.51317519 - samples/sec: 100.89 - lr: 0.100000\n","2021-05-02 13:06:39,184 epoch 8 - iter 540/2706 - loss 4.49418278 - samples/sec: 100.09 - lr: 0.100000\n","2021-05-02 13:07:18,305 epoch 8 - iter 810/2706 - loss 4.48064368 - samples/sec: 103.56 - lr: 0.100000\n","2021-05-02 13:07:57,239 epoch 8 - iter 1080/2706 - loss 4.44039263 - samples/sec: 104.06 - lr: 0.100000\n","2021-05-02 13:08:36,980 epoch 8 - iter 1350/2706 - loss 4.44717322 - samples/sec: 101.94 - lr: 0.100000\n","2021-05-02 13:09:17,427 epoch 8 - iter 1620/2706 - loss 4.47458982 - samples/sec: 100.16 - lr: 0.100000\n","2021-05-02 13:09:57,155 epoch 8 - iter 1890/2706 - loss 4.46117563 - samples/sec: 101.98 - lr: 0.100000\n","2021-05-02 13:10:36,897 epoch 8 - iter 2160/2706 - loss 4.46391580 - samples/sec: 101.94 - lr: 0.100000\n","2021-05-02 13:11:17,289 epoch 8 - iter 2430/2706 - loss 4.47177370 - samples/sec: 100.30 - lr: 0.100000\n","2021-05-02 13:11:57,215 epoch 8 - iter 2700/2706 - loss 4.46635061 - samples/sec: 101.47 - lr: 0.100000\n","2021-05-02 13:11:58,204 ----------------------------------------------------------------------------------------------------\n","2021-05-02 13:11:58,205 EPOCH 8 done: loss 4.4661 - lr 0.1000000\n","2021-05-02 13:12:11,467 DEV : loss 4.030393600463867 - score 0.813\n","2021-05-02 13:12:11,708 BAD EPOCHS (no improvement): 2\n","2021-05-02 13:12:11,713 ----------------------------------------------------------------------------------------------------\n","2021-05-02 13:12:51,532 epoch 9 - iter 270/2706 - loss 4.43212554 - samples/sec: 101.74 - lr: 0.100000\n","2021-05-02 13:13:31,764 epoch 9 - iter 540/2706 - loss 4.47364701 - samples/sec: 100.70 - lr: 0.100000\n","2021-05-02 13:14:10,449 epoch 9 - iter 810/2706 - loss 4.42441791 - samples/sec: 104.72 - lr: 0.100000\n","2021-05-02 13:14:51,156 epoch 9 - iter 1080/2706 - loss 4.45084132 - samples/sec: 99.52 - lr: 0.100000\n","2021-05-02 13:15:30,478 epoch 9 - iter 1350/2706 - loss 4.41499498 - samples/sec: 103.03 - lr: 0.100000\n","2021-05-02 13:16:09,207 epoch 9 - iter 1620/2706 - loss 4.39583980 - samples/sec: 104.60 - lr: 0.100000\n","2021-05-02 13:16:50,000 epoch 9 - iter 1890/2706 - loss 4.40087171 - samples/sec: 99.31 - lr: 0.100000\n","2021-05-02 13:17:29,609 epoch 9 - iter 2160/2706 - loss 4.40353297 - samples/sec: 102.28 - lr: 0.100000\n","2021-05-02 13:18:09,803 epoch 9 - iter 2430/2706 - loss 4.41041716 - samples/sec: 100.79 - lr: 0.100000\n","2021-05-02 13:18:50,218 epoch 9 - iter 2700/2706 - loss 4.41075859 - samples/sec: 100.24 - lr: 0.100000\n","2021-05-02 13:18:51,095 ----------------------------------------------------------------------------------------------------\n","2021-05-02 13:18:51,097 EPOCH 9 done: loss 4.4114 - lr 0.1000000\n","2021-05-02 13:19:04,367 DEV : loss 3.9666590690612793 - score 0.8168\n","2021-05-02 13:19:04,617 BAD EPOCHS (no improvement): 0\n","saving best model\n","2021-05-02 13:19:05,598 ----------------------------------------------------------------------------------------------------\n","2021-05-02 13:19:46,371 epoch 10 - iter 270/2706 - loss 4.31218224 - samples/sec: 99.37 - lr: 0.100000\n","2021-05-02 13:20:25,404 epoch 10 - iter 540/2706 - loss 4.30284852 - samples/sec: 103.79 - lr: 0.100000\n","2021-05-02 13:21:05,089 epoch 10 - iter 810/2706 - loss 4.33762449 - samples/sec: 102.08 - lr: 0.100000\n","2021-05-02 13:21:44,242 epoch 10 - iter 1080/2706 - loss 4.32031941 - samples/sec: 103.47 - lr: 0.100000\n","2021-05-02 13:22:24,706 epoch 10 - iter 1350/2706 - loss 4.34157610 - samples/sec: 100.12 - lr: 0.100000\n","2021-05-02 13:23:04,840 epoch 10 - iter 1620/2706 - loss 4.35614754 - samples/sec: 100.94 - lr: 0.100000\n","2021-05-02 13:23:43,764 epoch 10 - iter 1890/2706 - loss 4.36173812 - samples/sec: 104.08 - lr: 0.100000\n","2021-05-02 13:24:23,128 epoch 10 - iter 2160/2706 - loss 4.36416214 - samples/sec: 102.92 - lr: 0.100000\n","2021-05-02 13:25:03,202 epoch 10 - iter 2430/2706 - loss 4.37050698 - samples/sec: 101.09 - lr: 0.100000\n","2021-05-02 13:25:42,761 epoch 10 - iter 2700/2706 - loss 4.37150791 - samples/sec: 102.41 - lr: 0.100000\n","2021-05-02 13:25:43,750 ----------------------------------------------------------------------------------------------------\n","2021-05-02 13:25:43,752 EPOCH 10 done: loss 4.3724 - lr 0.1000000\n","2021-05-02 13:25:56,955 DEV : loss 4.018640041351318 - score 0.8135\n","2021-05-02 13:25:57,205 BAD EPOCHS (no improvement): 1\n","2021-05-02 13:25:57,210 ----------------------------------------------------------------------------------------------------\n","2021-05-02 13:26:38,200 epoch 11 - iter 270/2706 - loss 4.58104524 - samples/sec: 98.84 - lr: 0.100000\n","2021-05-02 13:27:17,716 epoch 11 - iter 540/2706 - loss 4.49384849 - samples/sec: 102.52 - lr: 0.100000\n","2021-05-02 13:27:57,556 epoch 11 - iter 810/2706 - loss 4.39810387 - samples/sec: 101.69 - lr: 0.100000\n","2021-05-02 13:28:36,751 epoch 11 - iter 1080/2706 - loss 4.39693293 - samples/sec: 103.36 - lr: 0.100000\n","2021-05-02 13:29:15,656 epoch 11 - iter 1350/2706 - loss 4.35662285 - samples/sec: 104.13 - lr: 0.100000\n","2021-05-02 13:29:55,650 epoch 11 - iter 1620/2706 - loss 4.35146320 - samples/sec: 101.30 - lr: 0.100000\n","2021-05-02 13:30:34,824 epoch 11 - iter 1890/2706 - loss 4.34929886 - samples/sec: 103.42 - lr: 0.100000\n","2021-05-02 13:31:14,143 epoch 11 - iter 2160/2706 - loss 4.35284153 - samples/sec: 103.04 - lr: 0.100000\n","2021-05-02 13:31:53,751 epoch 11 - iter 2430/2706 - loss 4.35901377 - samples/sec: 102.28 - lr: 0.100000\n","2021-05-02 13:32:33,927 epoch 11 - iter 2700/2706 - loss 4.35909932 - samples/sec: 100.84 - lr: 0.100000\n","2021-05-02 13:32:34,859 ----------------------------------------------------------------------------------------------------\n","2021-05-02 13:32:34,862 EPOCH 11 done: loss 4.3615 - lr 0.1000000\n","2021-05-02 13:32:48,061 DEV : loss 3.953094005584717 - score 0.8166\n","2021-05-02 13:32:48,304 BAD EPOCHS (no improvement): 2\n","2021-05-02 13:32:48,308 ----------------------------------------------------------------------------------------------------\n","2021-05-02 13:33:27,499 epoch 12 - iter 270/2706 - loss 4.14458566 - samples/sec: 103.38 - lr: 0.100000\n","2021-05-02 13:34:08,040 epoch 12 - iter 540/2706 - loss 4.22643569 - samples/sec: 99.93 - lr: 0.100000\n","2021-05-02 13:34:47,710 epoch 12 - iter 810/2706 - loss 4.31870341 - samples/sec: 102.12 - lr: 0.100000\n","2021-05-02 13:35:27,260 epoch 12 - iter 1080/2706 - loss 4.32557962 - samples/sec: 102.43 - lr: 0.100000\n","2021-05-02 13:36:06,742 epoch 12 - iter 1350/2706 - loss 4.33820631 - samples/sec: 102.61 - lr: 0.100000\n","2021-05-02 13:36:46,340 epoch 12 - iter 1620/2706 - loss 4.34142802 - samples/sec: 102.31 - lr: 0.100000\n","2021-05-02 13:37:26,601 epoch 12 - iter 1890/2706 - loss 4.33767156 - samples/sec: 100.62 - lr: 0.100000\n","2021-05-02 13:38:04,925 epoch 12 - iter 2160/2706 - loss 4.33640879 - samples/sec: 105.71 - lr: 0.100000\n","2021-05-02 13:38:43,801 epoch 12 - iter 2430/2706 - loss 4.32601599 - samples/sec: 104.21 - lr: 0.100000\n","2021-05-02 13:39:24,267 epoch 12 - iter 2700/2706 - loss 4.34152411 - samples/sec: 100.11 - lr: 0.100000\n","2021-05-02 13:39:25,233 ----------------------------------------------------------------------------------------------------\n","2021-05-02 13:39:25,235 EPOCH 12 done: loss 4.3404 - lr 0.1000000\n","2021-05-02 13:39:38,448 DEV : loss 3.9829187393188477 - score 0.8169\n","2021-05-02 13:39:38,693 BAD EPOCHS (no improvement): 0\n","saving best model\n","2021-05-02 13:39:39,684 ----------------------------------------------------------------------------------------------------\n","2021-05-02 13:40:20,263 epoch 13 - iter 270/2706 - loss 4.30260353 - samples/sec: 99.84 - lr: 0.100000\n","2021-05-02 13:40:59,774 epoch 13 - iter 540/2706 - loss 4.30134512 - samples/sec: 102.53 - lr: 0.100000\n","2021-05-02 13:41:38,374 epoch 13 - iter 810/2706 - loss 4.26647873 - samples/sec: 104.95 - lr: 0.100000\n","2021-05-02 13:42:17,698 epoch 13 - iter 1080/2706 - loss 4.30655606 - samples/sec: 103.02 - lr: 0.100000\n","2021-05-02 13:42:56,592 epoch 13 - iter 1350/2706 - loss 4.29911628 - samples/sec: 104.16 - lr: 0.100000\n","2021-05-02 13:43:36,488 epoch 13 - iter 1620/2706 - loss 4.32441218 - samples/sec: 101.54 - lr: 0.100000\n","2021-05-02 13:44:15,620 epoch 13 - iter 1890/2706 - loss 4.31740667 - samples/sec: 103.53 - lr: 0.100000\n","2021-05-02 13:44:53,565 epoch 13 - iter 2160/2706 - loss 4.28641383 - samples/sec: 106.76 - lr: 0.100000\n","2021-05-02 13:45:32,976 epoch 13 - iter 2430/2706 - loss 4.29403053 - samples/sec: 102.79 - lr: 0.100000\n","2021-05-02 13:46:12,758 epoch 13 - iter 2700/2706 - loss 4.30677691 - samples/sec: 101.83 - lr: 0.100000\n","2021-05-02 13:46:13,622 ----------------------------------------------------------------------------------------------------\n","2021-05-02 13:46:13,624 EPOCH 13 done: loss 4.3068 - lr 0.1000000\n","2021-05-02 13:46:28,966 DEV : loss 3.9587197303771973 - score 0.8164\n","2021-05-02 13:46:29,206 BAD EPOCHS (no improvement): 1\n","2021-05-02 13:46:29,210 ----------------------------------------------------------------------------------------------------\n","2021-05-02 13:47:07,742 epoch 14 - iter 270/2706 - loss 4.23069832 - samples/sec: 105.15 - lr: 0.100000\n","2021-05-02 13:47:48,207 epoch 14 - iter 540/2706 - loss 4.31710497 - samples/sec: 100.11 - lr: 0.100000\n","2021-05-02 13:48:27,167 epoch 14 - iter 810/2706 - loss 4.29889111 - samples/sec: 103.98 - lr: 0.100000\n","2021-05-02 13:49:06,426 epoch 14 - iter 1080/2706 - loss 4.27607655 - samples/sec: 103.19 - lr: 0.100000\n","2021-05-02 13:49:45,759 epoch 14 - iter 1350/2706 - loss 4.27113819 - samples/sec: 103.00 - lr: 0.100000\n","2021-05-02 13:50:24,909 epoch 14 - iter 1620/2706 - loss 4.27079721 - samples/sec: 103.48 - lr: 0.100000\n","2021-05-02 13:51:04,236 epoch 14 - iter 1890/2706 - loss 4.27149723 - samples/sec: 103.02 - lr: 0.100000\n","2021-05-02 13:51:42,693 epoch 14 - iter 2160/2706 - loss 4.25629823 - samples/sec: 105.34 - lr: 0.100000\n","2021-05-02 13:52:22,496 epoch 14 - iter 2430/2706 - loss 4.26216446 - samples/sec: 101.79 - lr: 0.100000\n","2021-05-02 13:53:02,243 epoch 14 - iter 2700/2706 - loss 4.27676581 - samples/sec: 101.93 - lr: 0.100000\n","2021-05-02 13:53:03,142 ----------------------------------------------------------------------------------------------------\n","2021-05-02 13:53:03,144 EPOCH 14 done: loss 4.2759 - lr 0.1000000\n","2021-05-02 13:53:16,303 DEV : loss 3.940789222717285 - score 0.8164\n","2021-05-02 13:53:16,546 BAD EPOCHS (no improvement): 2\n","2021-05-02 13:53:16,552 ----------------------------------------------------------------------------------------------------\n","2021-05-02 13:53:57,252 epoch 15 - iter 270/2706 - loss 4.32766599 - samples/sec: 99.55 - lr: 0.100000\n","2021-05-02 13:54:35,706 epoch 15 - iter 540/2706 - loss 4.22750424 - samples/sec: 105.35 - lr: 0.100000\n","2021-05-02 13:55:15,029 epoch 15 - iter 810/2706 - loss 4.22164775 - samples/sec: 103.02 - lr: 0.100000\n","2021-05-02 13:55:55,140 epoch 15 - iter 1080/2706 - loss 4.21349237 - samples/sec: 101.00 - lr: 0.100000\n","2021-05-02 13:56:35,885 epoch 15 - iter 1350/2706 - loss 4.24220749 - samples/sec: 99.43 - lr: 0.100000\n","2021-05-02 13:57:15,408 epoch 15 - iter 1620/2706 - loss 4.26002644 - samples/sec: 102.51 - lr: 0.100000\n","2021-05-02 13:57:54,506 epoch 15 - iter 1890/2706 - loss 4.26245092 - samples/sec: 103.62 - lr: 0.100000\n","2021-05-02 13:58:32,787 epoch 15 - iter 2160/2706 - loss 4.25122706 - samples/sec: 105.83 - lr: 0.100000\n","2021-05-02 13:59:12,141 epoch 15 - iter 2430/2706 - loss 4.25272736 - samples/sec: 102.94 - lr: 0.100000\n","2021-05-02 13:59:49,836 epoch 15 - iter 2700/2706 - loss 4.25197560 - samples/sec: 107.47 - lr: 0.100000\n","2021-05-02 13:59:50,682 ----------------------------------------------------------------------------------------------------\n","2021-05-02 13:59:50,684 EPOCH 15 done: loss 4.2504 - lr 0.1000000\n","2021-05-02 14:00:03,818 DEV : loss 3.9366414546966553 - score 0.8164\n","2021-05-02 14:00:04,064 BAD EPOCHS (no improvement): 3\n","2021-05-02 14:00:04,068 ----------------------------------------------------------------------------------------------------\n","2021-05-02 14:00:44,302 epoch 16 - iter 270/2706 - loss 4.19654861 - samples/sec: 100.70 - lr: 0.100000\n","2021-05-02 14:01:24,222 epoch 16 - iter 540/2706 - loss 4.27049439 - samples/sec: 101.48 - lr: 0.100000\n","2021-05-02 14:02:03,820 epoch 16 - iter 810/2706 - loss 4.24800841 - samples/sec: 102.31 - lr: 0.100000\n","2021-05-02 14:02:42,092 epoch 16 - iter 1080/2706 - loss 4.22259526 - samples/sec: 105.85 - lr: 0.100000\n","2021-05-02 14:03:21,788 epoch 16 - iter 1350/2706 - loss 4.21808079 - samples/sec: 102.06 - lr: 0.100000\n","2021-05-02 14:04:01,961 epoch 16 - iter 1620/2706 - loss 4.23468504 - samples/sec: 100.84 - lr: 0.100000\n","2021-05-02 14:04:41,313 epoch 16 - iter 1890/2706 - loss 4.23429485 - samples/sec: 102.95 - lr: 0.100000\n","2021-05-02 14:05:20,015 epoch 16 - iter 2160/2706 - loss 4.22522168 - samples/sec: 104.68 - lr: 0.100000\n","2021-05-02 14:05:59,462 epoch 16 - iter 2430/2706 - loss 4.22496478 - samples/sec: 102.70 - lr: 0.100000\n","2021-05-02 14:06:38,834 epoch 16 - iter 2700/2706 - loss 4.22754022 - samples/sec: 102.89 - lr: 0.100000\n","2021-05-02 14:06:39,748 ----------------------------------------------------------------------------------------------------\n","2021-05-02 14:06:39,750 EPOCH 16 done: loss 4.2289 - lr 0.1000000\n","2021-05-02 14:06:52,882 DEV : loss 3.9168648719787598 - score 0.8183\n","2021-05-02 14:06:53,120 BAD EPOCHS (no improvement): 0\n","saving best model\n","2021-05-02 14:06:54,111 ----------------------------------------------------------------------------------------------------\n","2021-05-02 14:07:35,899 epoch 17 - iter 270/2706 - loss 4.35449482 - samples/sec: 96.96 - lr: 0.100000\n","2021-05-02 14:08:15,062 epoch 17 - iter 540/2706 - loss 4.25553401 - samples/sec: 103.44 - lr: 0.100000\n","2021-05-02 14:08:54,846 epoch 17 - iter 810/2706 - loss 4.24579495 - samples/sec: 101.83 - lr: 0.100000\n","2021-05-02 14:09:34,624 epoch 17 - iter 1080/2706 - loss 4.21917580 - samples/sec: 101.84 - lr: 0.100000\n","2021-05-02 14:10:13,554 epoch 17 - iter 1350/2706 - loss 4.21187479 - samples/sec: 104.06 - lr: 0.100000\n","2021-05-02 14:10:53,445 epoch 17 - iter 1620/2706 - loss 4.21201525 - samples/sec: 101.56 - lr: 0.100000\n","2021-05-02 14:11:32,208 epoch 17 - iter 1890/2706 - loss 4.21315832 - samples/sec: 104.51 - lr: 0.100000\n","2021-05-02 14:12:11,679 epoch 17 - iter 2160/2706 - loss 4.21908241 - samples/sec: 102.64 - lr: 0.100000\n","2021-05-02 14:12:50,496 epoch 17 - iter 2430/2706 - loss 4.21871998 - samples/sec: 104.36 - lr: 0.100000\n","2021-05-02 14:13:29,332 epoch 17 - iter 2700/2706 - loss 4.20916387 - samples/sec: 104.32 - lr: 0.100000\n","2021-05-02 14:13:30,467 ----------------------------------------------------------------------------------------------------\n","2021-05-02 14:13:30,469 EPOCH 17 done: loss 4.2116 - lr 0.1000000\n","2021-05-02 14:13:43,721 DEV : loss 3.9330925941467285 - score 0.8167\n","2021-05-02 14:13:43,960 BAD EPOCHS (no improvement): 1\n","2021-05-02 14:13:43,965 ----------------------------------------------------------------------------------------------------\n","2021-05-02 14:14:23,260 epoch 18 - iter 270/2706 - loss 4.09584623 - samples/sec: 103.11 - lr: 0.100000\n","2021-05-02 14:15:01,684 epoch 18 - iter 540/2706 - loss 4.10282013 - samples/sec: 105.43 - lr: 0.100000\n","2021-05-02 14:15:41,400 epoch 18 - iter 810/2706 - loss 4.09345814 - samples/sec: 102.00 - lr: 0.100000\n","2021-05-02 14:16:20,920 epoch 18 - iter 1080/2706 - loss 4.11722780 - samples/sec: 102.51 - lr: 0.100000\n","2021-05-02 14:16:59,117 epoch 18 - iter 1350/2706 - loss 4.12959996 - samples/sec: 106.06 - lr: 0.100000\n","2021-05-02 14:17:37,679 epoch 18 - iter 1620/2706 - loss 4.15437921 - samples/sec: 105.06 - lr: 0.100000\n","2021-05-02 14:18:18,268 epoch 18 - iter 1890/2706 - loss 4.16830753 - samples/sec: 99.81 - lr: 0.100000\n","2021-05-02 14:18:57,990 epoch 18 - iter 2160/2706 - loss 4.15697017 - samples/sec: 101.99 - lr: 0.100000\n","2021-05-02 14:19:38,069 epoch 18 - iter 2430/2706 - loss 4.17924955 - samples/sec: 101.08 - lr: 0.100000\n","2021-05-02 14:20:17,915 epoch 18 - iter 2700/2706 - loss 4.18981935 - samples/sec: 101.67 - lr: 0.100000\n","2021-05-02 14:20:18,724 ----------------------------------------------------------------------------------------------------\n","2021-05-02 14:20:18,725 EPOCH 18 done: loss 4.1888 - lr 0.1000000\n","2021-05-02 14:20:31,872 DEV : loss 3.908781051635742 - score 0.8196\n","2021-05-02 14:20:32,115 BAD EPOCHS (no improvement): 0\n","saving best model\n","2021-05-02 14:20:33,173 ----------------------------------------------------------------------------------------------------\n","2021-05-02 14:21:14,127 epoch 19 - iter 270/2706 - loss 4.12991924 - samples/sec: 98.93 - lr: 0.100000\n","2021-05-02 14:21:53,485 epoch 19 - iter 540/2706 - loss 4.15550392 - samples/sec: 102.93 - lr: 0.100000\n","2021-05-02 14:22:33,313 epoch 19 - iter 810/2706 - loss 4.19724336 - samples/sec: 101.71 - lr: 0.100000\n","2021-05-02 14:23:12,294 epoch 19 - iter 1080/2706 - loss 4.15748898 - samples/sec: 103.93 - lr: 0.100000\n","2021-05-02 14:23:51,714 epoch 19 - iter 1350/2706 - loss 4.13568075 - samples/sec: 102.77 - lr: 0.100000\n","2021-05-02 14:24:29,653 epoch 19 - iter 1620/2706 - loss 4.11842746 - samples/sec: 106.78 - lr: 0.100000\n","2021-05-02 14:25:09,465 epoch 19 - iter 1890/2706 - loss 4.14167989 - samples/sec: 101.76 - lr: 0.100000\n","2021-05-02 14:25:48,454 epoch 19 - iter 2160/2706 - loss 4.14777345 - samples/sec: 103.91 - lr: 0.100000\n","2021-05-02 14:26:27,056 epoch 19 - iter 2430/2706 - loss 4.15616378 - samples/sec: 104.95 - lr: 0.100000\n","2021-05-02 14:27:07,091 epoch 19 - iter 2700/2706 - loss 4.16579477 - samples/sec: 101.19 - lr: 0.100000\n","2021-05-02 14:27:07,967 ----------------------------------------------------------------------------------------------------\n","2021-05-02 14:27:07,969 EPOCH 19 done: loss 4.1670 - lr 0.1000000\n","2021-05-02 14:27:21,104 DEV : loss 3.89491605758667 - score 0.8188\n","2021-05-02 14:27:21,344 BAD EPOCHS (no improvement): 1\n","2021-05-02 14:27:21,348 ----------------------------------------------------------------------------------------------------\n","2021-05-02 14:28:01,523 epoch 20 - iter 270/2706 - loss 4.01598301 - samples/sec: 100.85 - lr: 0.100000\n","2021-05-02 14:28:41,110 epoch 20 - iter 540/2706 - loss 4.09138427 - samples/sec: 102.34 - lr: 0.100000\n","2021-05-02 14:29:21,073 epoch 20 - iter 810/2706 - loss 4.13668458 - samples/sec: 101.37 - lr: 0.100000\n","2021-05-02 14:30:00,076 epoch 20 - iter 1080/2706 - loss 4.14736734 - samples/sec: 103.87 - lr: 0.100000\n","2021-05-02 14:30:39,565 epoch 20 - iter 1350/2706 - loss 4.14132868 - samples/sec: 102.59 - lr: 0.100000\n","2021-05-02 14:31:17,919 epoch 20 - iter 1620/2706 - loss 4.12591015 - samples/sec: 105.63 - lr: 0.100000\n","2021-05-02 14:31:57,339 epoch 20 - iter 1890/2706 - loss 4.14108358 - samples/sec: 102.77 - lr: 0.100000\n","2021-05-02 14:32:36,898 epoch 20 - iter 2160/2706 - loss 4.14485900 - samples/sec: 102.41 - lr: 0.100000\n","2021-05-02 14:33:17,472 epoch 20 - iter 2430/2706 - loss 4.15694851 - samples/sec: 99.84 - lr: 0.100000\n","2021-05-02 14:33:55,375 epoch 20 - iter 2700/2706 - loss 4.16112027 - samples/sec: 106.88 - lr: 0.100000\n","2021-05-02 14:33:56,348 ----------------------------------------------------------------------------------------------------\n","2021-05-02 14:33:56,350 EPOCH 20 done: loss 4.1610 - lr 0.1000000\n","2021-05-02 14:34:09,522 DEV : loss 3.9048550128936768 - score 0.8178\n","2021-05-02 14:34:09,763 BAD EPOCHS (no improvement): 2\n","2021-05-02 14:34:10,796 ----------------------------------------------------------------------------------------------------\n","2021-05-02 14:34:10,799 Testing using best model ...\n","2021-05-02 14:34:10,804 loading file /content/gdrive/MyDrive/NLP projects/NER projects/project 1/best-model.pt\n","2021-05-02 14:36:07,831 0.8495\t0.8010\t0.8246\n","2021-05-02 14:36:07,832 \n","Results:\n","- F1-score (micro) 0.8246\n","- F1-score (macro) 0.6062\n","\n","By class:\n",".          tp: 3631 - fp: 0 - fn: 0 - precision: 1.0000 - recall: 1.0000 - f1-score: 1.0000\n","art        tp: 0 - fp: 3 - fn: 47 - precision: 0.0000 - recall: 0.0000 - f1-score: 0.0000\n","eve        tp: 9 - fp: 3 - fn: 33 - precision: 0.7500 - recall: 0.2143 - f1-score: 0.3333\n","geo        tp: 2734 - fp: 856 - fn: 574 - precision: 0.7616 - recall: 0.8265 - f1-score: 0.7927\n","gpe        tp: 1057 - fp: 100 - fn: 124 - precision: 0.9136 - recall: 0.8950 - f1-score: 0.9042\n","nat        tp: 3 - fp: 3 - fn: 18 - precision: 0.5000 - recall: 0.1429 - f1-score: 0.2222\n","org        tp: 1342 - fp: 415 - fn: 1239 - precision: 0.7638 - recall: 0.5200 - f1-score: 0.6187\n","per        tp: 2028 - fp: 667 - fn: 379 - precision: 0.7525 - recall: 0.8425 - f1-score: 0.7950\n","tim        tp: 1284 - fp: 94 - fn: 589 - precision: 0.9318 - recall: 0.6855 - f1-score: 0.7899\n","2021-05-02 14:36:07,835 ----------------------------------------------------------------------------------------------------\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"_kX_rdjQ8JBj"},"source":[""],"execution_count":null,"outputs":[]}]}