text_classification_with_BERT/Classification_with_Simple_Transformers.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-10T10:47:43.528395Z",
     "start_time": "2020-03-10T10:47:43.526128Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "import re\n",
    "from simpletransformers.classification import ClassificationModel\n",
    "import sklearn\n",
    "import itertools\n",
    "import emoji"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-10T10:47:43.554471Z",
     "start_time": "2020-03-10T10:47:43.529670Z"
    }
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"./train.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-10T10:47:43.557622Z",
     "start_time": "2020-03-10T10:47:43.555745Z"
    }
   },
   "outputs": [],
   "source": [
    "columns_drop = ['keyword','location']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-10T10:47:43.563226Z",
     "start_time": "2020-03-10T10:47:43.558742Z"
    }
   },
   "outputs": [],
   "source": [
    "df.drop(columns=columns_drop,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-10T10:47:43.571001Z",
     "start_time": "2020-03-10T10:47:43.564283Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Our Deeds are the Reason of this #earthquake M...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>Forest fire near La Ronge Sask. Canada</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>All residents asked to 'shelter in place' are ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>13,000 people receive #wildfires evacuation or...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>Just got sent this photo from Ruby #Alaska as ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id                                               text  target\n",
       "0   1  Our Deeds are the Reason of this #earthquake M...       1\n",
       "1   4             Forest fire near La Ronge Sask. Canada       1\n",
       "2   5  All residents asked to 'shelter in place' are ...       1\n",
       "3   6  13,000 people receive #wildfires evacuation or...       1\n",
       "4   7  Just got sent this photo from Ruby #Alaska as ...       1"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-10T13:28:51.112787Z",
     "start_time": "2020-03-10T13:28:51.105736Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4342, 3)"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fake_tweets = df[df.target == 0]\n",
    "fake_tweets.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-10T13:29:28.887073Z",
     "start_time": "2020-03-10T13:29:28.876505Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>23</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>24</td>\n",
       "      <td>fruits</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>25</td>\n",
       "      <td>Summer lovely</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>26</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>28</td>\n",
       "      <td>gooaal</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>31</td>\n",
       "      <td>ridiculous</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>32</td>\n",
       "      <td>London</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>33</td>\n",
       "      <td>skiing</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>34</td>\n",
       "      <td>wonderful</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>36</td>\n",
       "      <td>LOOL</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>37</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>38</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>39</td>\n",
       "      <td>girlfriend</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>40</td>\n",
       "      <td>Cool</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>41</td>\n",
       "      <td>pasta</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>44</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>49</td>\n",
       "      <td>always bring heavy metal</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>52</td>\n",
       "      <td>Crying ablaze</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>53</td>\n",
       "      <td>NIGHT ABLAZE</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>54</td>\n",
       "      <td>built around acquisitions doubt ablaze season</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>57</td>\n",
       "      <td>Ablaze</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>59</td>\n",
       "      <td>Check these</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>61</td>\n",
       "      <td>outside ablaze alive inside</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>62</td>\n",
       "      <td>awesome visiting office ancop ablaze Thanks ta...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>63</td>\n",
       "      <td>SOO PUMPED ABLAZE</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>64</td>\n",
       "      <td>wanted Chicago ablaze preaching hotel</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>65</td>\n",
       "      <td>gained followers stats</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>67</td>\n",
       "      <td>Building perfect tracklist leave streets ablaze</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>68</td>\n",
       "      <td>Check these</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>71</td>\n",
       "      <td>First night retainers quite weird Better every...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>441</th>\n",
       "      <td>640</td>\n",
       "      <td>court after</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>442</th>\n",
       "      <td>641</td>\n",
       "      <td>should</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>443</th>\n",
       "      <td>642</td>\n",
       "      <td>Owner Chicago Admits Arson Scheme Frank Elliot...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>444</th>\n",
       "      <td>643</td>\n",
       "      <td>Trusting terrorism inviting arsonist brigade T...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>447</th>\n",
       "      <td>649</td>\n",
       "      <td>Credit inspiring rediscover fantabulous</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>450</th>\n",
       "      <td>653</td>\n",
       "      <td>stans really think Heart Attack million copies</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>453</th>\n",
       "      <td>656</td>\n",
       "      <td>funniest about twitter feminists attack 4Head</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>463</th>\n",
       "      <td>669</td>\n",
       "      <td>heart attack because thought worry Rocket</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>464</th>\n",
       "      <td>671</td>\n",
       "      <td>gonna kinda ready attack Senior</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>465</th>\n",
       "      <td>672</td>\n",
       "      <td>diamond graveyard shift attack defend right ha...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>469</th>\n",
       "      <td>679</td>\n",
       "      <td>thanks heart attack</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>476</th>\n",
       "      <td>686</td>\n",
       "      <td>everyone ignoranceshe Latinoand benothing more...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>477</th>\n",
       "      <td>687</td>\n",
       "      <td>Heart disease prevention about secondhand smoke</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>479</th>\n",
       "      <td>689</td>\n",
       "      <td>Attack Titan</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>481</th>\n",
       "      <td>691</td>\n",
       "      <td>anxiety attack</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>482</th>\n",
       "      <td>697</td>\n",
       "      <td>attacked pugprobs</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>484</th>\n",
       "      <td>700</td>\n",
       "      <td>asked attacked julie asked frail</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485</th>\n",
       "      <td>702</td>\n",
       "      <td>attacked</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>486</th>\n",
       "      <td>704</td>\n",
       "      <td>feeling attacked</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>487</th>\n",
       "      <td>705</td>\n",
       "      <td>again black White black getting attacked</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>488</th>\n",
       "      <td>706</td>\n",
       "      <td>believe fucking female going somehow claim off...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>496</th>\n",
       "      <td>719</td>\n",
       "      <td>Remember Patrick attacked driver</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>497</th>\n",
       "      <td>720</td>\n",
       "      <td>feeling attacked</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>498</th>\n",
       "      <td>721</td>\n",
       "      <td>Troll Rivals Never Literally Abused Attacked T...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>501</th>\n",
       "      <td>726</td>\n",
       "      <td>worst person Questioning julie attacked empathy</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>503</th>\n",
       "      <td>728</td>\n",
       "      <td>needs chill answer calmly being attacked</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>507</th>\n",
       "      <td>732</td>\n",
       "      <td>attacked Robot earned total 6615434 satoshis r...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>509</th>\n",
       "      <td>735</td>\n",
       "      <td>these tweets really attacked</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>515</th>\n",
       "      <td>744</td>\n",
       "      <td>attacked</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>517</th>\n",
       "      <td>746</td>\n",
       "      <td>WeLoveLA NHLDucks Avalanche Defense Match Loui...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>300 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      id                                               text  target\n",
       "15    23                                                          0\n",
       "16    24                                             fruits       0\n",
       "17    25                                      Summer lovely       0\n",
       "18    26                                                          0\n",
       "19    28                                             gooaal       0\n",
       "20    31                                         ridiculous       0\n",
       "21    32                                             London       0\n",
       "22    33                                             skiing       0\n",
       "23    34                                          wonderful       0\n",
       "24    36                                               LOOL       0\n",
       "25    37                                                          0\n",
       "26    38                                                          0\n",
       "27    39                                         girlfriend       0\n",
       "28    40                                               Cool       0\n",
       "29    41                                              pasta       0\n",
       "30    44                                                          0\n",
       "32    49                           always bring heavy metal       0\n",
       "34    52                                      Crying ablaze       0\n",
       "35    53                                       NIGHT ABLAZE       0\n",
       "36    54      built around acquisitions doubt ablaze season       0\n",
       "39    57                                             Ablaze       0\n",
       "40    59                                        Check these       0\n",
       "41    61                        outside ablaze alive inside       0\n",
       "42    62  awesome visiting office ancop ablaze Thanks ta...       0\n",
       "43    63                                  SOO PUMPED ABLAZE       0\n",
       "44    64              wanted Chicago ablaze preaching hotel       0\n",
       "45    65                             gained followers stats       0\n",
       "47    67    Building perfect tracklist leave streets ablaze       0\n",
       "48    68                                        Check these       0\n",
       "49    71  First night retainers quite weird Better every...       0\n",
       "..   ...                                                ...     ...\n",
       "441  640                                        court after       0\n",
       "442  641                                             should       0\n",
       "443  642  Owner Chicago Admits Arson Scheme Frank Elliot...       0\n",
       "444  643  Trusting terrorism inviting arsonist brigade T...       0\n",
       "447  649            Credit inspiring rediscover fantabulous       0\n",
       "450  653     stans really think Heart Attack million copies       0\n",
       "453  656      funniest about twitter feminists attack 4Head       0\n",
       "463  669          heart attack because thought worry Rocket       0\n",
       "464  671                    gonna kinda ready attack Senior       0\n",
       "465  672  diamond graveyard shift attack defend right ha...       0\n",
       "469  679                                thanks heart attack       0\n",
       "476  686  everyone ignoranceshe Latinoand benothing more...       0\n",
       "477  687    Heart disease prevention about secondhand smoke       0\n",
       "479  689                                       Attack Titan       0\n",
       "481  691                                     anxiety attack       0\n",
       "482  697                                  attacked pugprobs       0\n",
       "484  700                   asked attacked julie asked frail       0\n",
       "485  702                                           attacked       0\n",
       "486  704                                   feeling attacked       0\n",
       "487  705           again black White black getting attacked       0\n",
       "488  706  believe fucking female going somehow claim off...       0\n",
       "496  719                   Remember Patrick attacked driver       0\n",
       "497  720                                   feeling attacked       0\n",
       "498  721  Troll Rivals Never Literally Abused Attacked T...       0\n",
       "501  726    worst person Questioning julie attacked empathy       0\n",
       "503  728           needs chill answer calmly being attacked       0\n",
       "507  732  attacked Robot earned total 6615434 satoshis r...       0\n",
       "509  735                       these tweets really attacked       0\n",
       "515  744                                           attacked       0\n",
       "517  746  WeLoveLA NHLDucks Avalanche Defense Match Loui...       0\n",
       "\n",
       "[300 rows x 3 columns]"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fake_tweets.head(300)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Defining contractions to clean the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-10T10:47:43.578104Z",
     "start_time": "2020-03-10T10:47:43.571964Z"
    }
   },
   "outputs": [],
   "source": [
    "contractions = { \n",
    "\"ain't\": \"am not\",\n",
    "\"aren't\": \"are not\",\n",
    "\"can't\": \"cannot\",\n",
    "\"can't've\": \"cannot have\",\n",
    "\"'cause\": \"because\",\n",
    "\"could've\": \"could have\",\n",
    "\"couldn't\": \"could not\",\n",
    "\"couldn't've\": \"could not have\",\n",
    "\"didn't\": \"did not\",\n",
    "\"doesn't\": \"does not\",\n",
    "\"don't\": \"do not\",\n",
    "\"hadn't\": \"had not\",\n",
    "\"hadn't've\": \"had not have\",\n",
    "\"hasn't\": \"has not\",\n",
    "\"haven't\": \"have not\",\n",
    "\"he'd\": \"he would\",\n",
    "\"he'd've\": \"he would have\",\n",
    "\"he'll\": \"he will\",\n",
    "\"he's\": \"he is\",\n",
    "\"how'd\": \"how did\",\n",
    "\"how'll\": \"how will\",\n",
    "\"how's\": \"how is\",\n",
    "\"i'd\": \"i would\",\n",
    "\"i'll\": \"i will\",\n",
    "\"i'm\": \"i am\",\n",
    "\"i've\": \"i have\",\n",
    "\"isn't\": \"is not\",\n",
    "\"it'd\": \"it would\",\n",
    "\"it'll\": \"it will\",\n",
    "\"it's\": \"it is\",\n",
    "\"let's\": \"let us\",\n",
    "\"ma'am\": \"madam\",\n",
    "\"mayn't\": \"may not\",\n",
    "\"might've\": \"might have\",\n",
    "\"mightn't\": \"might not\",\n",
    "\"must've\": \"must have\",\n",
    "\"mustn't\": \"must not\",\n",
    "\"needn't\": \"need not\",\n",
    "\"oughtn't\": \"ought not\",\n",
    "\"shan't\": \"shall not\",\n",
    "\"sha'n't\": \"shall not\",\n",
    "\"she'd\": \"she would\",\n",
    "\"she'll\": \"she will\",\n",
    "\"she's\": \"she is\",\n",
    "\"should've\": \"should have\",\n",
    "\"shouldn't\": \"should not\",\n",
    "\"that'd\": \"that would\",\n",
    "\"that's\": \"that is\",\n",
    "\"there'd\": \"there had\",\n",
    "\"there's\": \"there is\",\n",
    "\"they'd\": \"they would\",\n",
    "\"they'll\": \"they will\",\n",
    "\"they're\": \"they are\",\n",
    "\"they've\": \"they have\",\n",
    "\"wasn't\": \"was not\",\n",
    "\"we'd\": \"we would\",\n",
    "\"we'll\": \"we will\",\n",
    "\"we're\": \"we are\",\n",
    "\"we've\": \"we have\",\n",
    "\"weren't\": \"were not\",\n",
    "\"what'll\": \"what will\",\n",
    "\"what're\": \"what are\",\n",
    "\"what's\": \"what is\",\n",
    "\"what've\": \"what have\",\n",
    "\"where'd\": \"where did\",\n",
    "\"where's\": \"where is\",\n",
    "\"who'll\": \"who will\",\n",
    "\"who's\": \"who is\",\n",
    "\"won't\": \"will not\",\n",
    "\"wouldn't\": \"would not\",\n",
    "\"you'd\": \"you would\",\n",
    "\"you'll\": \"you will\",\n",
    "\"you're\": \"you are\",\n",
    "\"thx\"   : \"thanks\"\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-10T10:47:43.584982Z",
     "start_time": "2020-03-10T10:47:43.579129Z"
    }
   },
   "outputs": [],
   "source": [
    "def remove_contractions(text):\n",
    "    return contractions[text.lower()] if text.lower() in contractions.keys() else text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-10T10:47:43.609732Z",
     "start_time": "2020-03-10T10:47:43.586510Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Our Deeds are the Reason of this #earthquake M...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>Forest fire near La Ronge Sask. Canada</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>All residents asked to 'shelter in place' are ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>13,000 people receive #wildfires evacuation or...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>Just got sent this photo from Ruby #Alaska as ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id                                               text  target\n",
       "0   1  Our Deeds are the Reason of this #earthquake M...       1\n",
       "1   4             Forest fire near La Ronge Sask. Canada       1\n",
       "2   5  All residents asked to 'shelter in place' are ...       1\n",
       "3   6  13,000 people receive #wildfires evacuation or...       1\n",
       "4   7  Just got sent this photo from Ruby #Alaska as ...       1"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['text']=df['text'].apply(remove_contractions)\n",
    "df.head()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Clean the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-10T10:47:43.619202Z",
     "start_time": "2020-03-10T10:47:43.611278Z"
    }
   },
   "outputs": [],
   "source": [
    "def clean_dataset(text):\n",
    "    # Remove hashtag while keeping hashtag text\n",
    "    text = re.sub(r'#','', text)\n",
    "    # Remove HTML special entities (e.g. &amp;)\n",
    "    text = re.sub(r'\\&\\w*;', '', text)\n",
    "    # Remove tickers\n",
    "    text = re.sub(r'\\$\\w*', '', text)\n",
    "    # Remove hyperlinks\n",
    "    text = re.sub(r'https?:\\/\\/.*\\/\\w*', '', text)\n",
    "    # Remove whitespace (including new line characters)\n",
    "    text = re.sub(r'\\s\\s+','', text)\n",
    "    text = re.sub(r'[ ]{2, }',' ',text)\n",
    "    # Remove URL, RT, mention(@)\n",
    "    text=  re.sub(r'http(\\S)+', '',text)\n",
    "    text=  re.sub(r'http ...', '',text)\n",
    "    text=  re.sub(r'(RT|rt)[ ]*@[ ]*[\\S]+','',text)\n",
    "    text=  re.sub(r'RT[ ]?@','',text)\n",
    "    text = re.sub(r'@[\\S]+','',text)\n",
    "    # Remove words with 4 or fewer letters\n",
    "    text = re.sub(r'\\b\\w{1,4}\\b', '', text)\n",
    "    #&, < and >\n",
    "    text = re.sub(r'&amp;?', 'and',text)\n",
    "    text = re.sub(r'&lt;','<',text)\n",
    "    text = re.sub(r'&gt;','>',text)\n",
    "    # Remove characters beyond Basic Multilingual Plane (BMP) of Unicode:\n",
    "    text= ''.join(c for c in text if c <= '\\uFFFF') \n",
    "    text = text.strip()\n",
    "    # Remove misspelling words\n",
    "    text = ''.join(''.join(s)[:2] for _, s in itertools.groupby(text))\n",
    "    # Remove emoji\n",
    "    text = emoji.demojize(text)\n",
    "    text = text.replace(\":\",\" \")\n",
    "    text = ' '.join(text.split()) \n",
    "    text = re.sub(\"([^\\x00-\\x7F])+\",\" \",text)\n",
    "    # Remove Mojibake (also extra spaces)\n",
    "    text = ' '.join(re.sub(\"[^\\u4e00-\\u9fa5\\u0030-\\u0039\\u0041-\\u005a\\u0061-\\u007a]\", \" \", text).split())\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-10T10:47:44.725957Z",
     "start_time": "2020-03-10T10:47:43.620437Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Deeds Reason earthquake ALLAH Forgive</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>Forest Ronge Canada</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>residents asked shelter place being notified o...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>people receive wildfires evacuation orders Cal...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>photo Alaska smoke wildfires pours school</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id                                               text  target\n",
       "0   1              Deeds Reason earthquake ALLAH Forgive       1\n",
       "1   4                                Forest Ronge Canada       1\n",
       "2   5  residents asked shelter place being notified o...       1\n",
       "3   6  people receive wildfires evacuation orders Cal...       1\n",
       "4   7          photo Alaska smoke wildfires pours school       1"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['text'] =df['text'].apply(clean_dataset)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-10T10:47:44.729250Z",
     "start_time": "2020-03-10T10:47:44.726983Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7613, 3)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Split the training and validation set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-10T10:47:44.736858Z",
     "start_time": "2020-03-10T10:47:44.730155Z"
    }
   },
   "outputs": [],
   "source": [
    "X_train_clean, X_test_clean, y_train_clean, y_test_clean = train_test_split(df['text'], df['target'], test_size=0.20, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-10T10:47:44.744448Z",
     "start_time": "2020-03-10T10:47:44.737800Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of training data set:  (6090, 2)\n",
      "View of data set:                                                     text  target\n",
      "4996  Courageous honest analysis Atomic Hiroshima70 ...       1\n",
      "3263          shame became engulfed flames boycottBears       0\n",
      "4907  rescind medals honor given soldiers Massacre W...       1\n",
      "2855  Worried about drought might affect Extreme Wea...       1\n",
      "4716                           BlastPower PantherAttack       0\n"
     ]
    }
   ],
   "source": [
    "train_df_clean = pd.concat([X_train_clean, y_train_clean], axis=1)\n",
    "print(\"Shape of training data set: \", train_df_clean.shape)\n",
    "print(\"View of data set: \", train_df_clean.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-10T10:47:44.749285Z",
     "start_time": "2020-03-10T10:47:44.745590Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of Eval data set:  (1523, 2)\n"
     ]
    }
   ],
   "source": [
    "eval_df_clean = pd.concat([X_test_clean, y_test_clean], axis=1)\n",
    "print(\"Shape of Eval data set: \", eval_df_clean.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### BERT Model Training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Set up the train arguments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-10T10:47:44.754966Z",
     "start_time": "2020-03-10T10:47:44.750323Z"
    }
   },
   "outputs": [],
   "source": [
    "train_args = {\n",
    "    'evaluate_during_training': True,\n",
    "    'logging_steps': 100,\n",
    "    'num_train_epochs': 2,\n",
    "    'evaluate_during_training_steps': 100,\n",
    "    'save_eval_checkpoints': False,\n",
    "    'train_batch_size': 32,\n",
    "    'eval_batch_size': 64,\n",
    "    'overwrite_output_dir': True,\n",
    "    'fp16': False,\n",
    "    'wandb_project': \"visualization-demo\"\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-10T10:47:47.439417Z",
     "start_time": "2020-03-10T10:47:44.755821Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 06:47:44.935707 140666048968448 configuration_utils.py:256] loading configuration file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-config.json from cache at /home/priya/.cache/torch/transformers/b945b69218e98b3e2c95acf911789741307dec43c698d35fad11c1ae28bda352.3d5adf10d3445c36ce131f4c6416aa62e9b58e1af56b97664773f4858a46286e\n",
      "I0310 06:47:44.936987 140666048968448 configuration_utils.py:292] Model config BertConfig {\n",
      "  \"architectures\": [\n",
      "    \"BertForMaskedLM\"\n",
      "  ],\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"bos_token_id\": null,\n",
      "  \"do_sample\": false,\n",
      "  \"eos_token_ids\": null,\n",
      "  \"finetuning_task\": null,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"id2label\": {\n",
      "    \"0\": \"LABEL_0\",\n",
      "    \"1\": \"LABEL_1\"\n",
      "  },\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"is_decoder\": false,\n",
      "  \"label2id\": {\n",
      "    \"LABEL_0\": 0,\n",
      "    \"LABEL_1\": 1\n",
      "  },\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"length_penalty\": 1.0,\n",
      "  \"max_length\": 20,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"model_type\": \"bert\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_beams\": 1,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"num_labels\": 2,\n",
      "  \"num_return_sequences\": 1,\n",
      "  \"output_attentions\": false,\n",
      "  \"output_hidden_states\": false,\n",
      "  \"output_past\": true,\n",
      "  \"pad_token_id\": null,\n",
      "  \"pruned_heads\": {},\n",
      "  \"repetition_penalty\": 1.0,\n",
      "  \"temperature\": 1.0,\n",
      "  \"top_k\": 50,\n",
      "  \"top_p\": 1.0,\n",
      "  \"torchscript\": false,\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"use_bfloat16\": false,\n",
      "  \"vocab_size\": 28996\n",
      "}\n",
      "\n",
      "I0310 06:47:45.110995 140666048968448 modeling_utils.py:461] loading weights file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-pytorch_model.bin from cache at /home/priya/.cache/torch/transformers/35d8b9d36faaf46728a0192d82bf7d00137490cd6074e8500778afed552a67e5.3fadbea36527ae472139fe84cddaa65454d7429f12d543d80bfc3ad70de55ac2\n",
      "I0310 06:47:47.223544 140666048968448 modeling_utils.py:546] Weights of BertForSequenceClassification not initialized from pretrained model: ['classifier.weight', 'classifier.bias']\n",
      "I0310 06:47:47.224052 140666048968448 modeling_utils.py:552] Weights from pretrained model not used in BertForSequenceClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias']\n",
      "I0310 06:47:47.420420 140666048968448 tokenization_utils.py:501] loading file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-vocab.txt from cache at /home/priya/.cache/torch/transformers/5e8a2b4893d13790ed4150ca1906be5f7a03d6c4ddf62296c383f6db42814db2.e13dbb970cb325137104fb2e5f36fe865f27746c6b526f6352861b1980eb80b1\n"
     ]
    }
   ],
   "source": [
    "model_BERT = ClassificationModel('bert', 'bert-base-cased', num_labels=2, use_cuda=True, cuda_device=0, args=train_args)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-10T10:50:43.143978Z",
     "start_time": "2020-03-10T10:47:47.440443Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Converting to features started. Cache is not used.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dfeae90860124adda12dceaecf6960e8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=6090), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f4d969dc91b84f8eb97270dbb69027c6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=2, style=ProgressStyle(description_width='initial…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "                Logging results to <a href=\"https://wandb.com\" target=\"_blank\">Weights & Biases</a> <a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">(Documentation)</a>.<br/>\n",
       "                Project page: <a href=\"https://app.wandb.ai/pdwivedi/visualization-demo\" target=\"_blank\">https://app.wandb.ai/pdwivedi/visualization-demo</a><br/>\n",
       "                Run page: <a href=\"https://app.wandb.ai/pdwivedi/visualization-demo/runs/hxlq47z4\" target=\"_blank\">https://app.wandb.ai/pdwivedi/visualization-demo/runs/hxlq47z4</a><br/>\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 06:47:49.057393 140666048968448 run_manager.py:924] system metrics and metadata threads started\n",
      "I0310 06:47:49.059631 140666048968448 run_manager.py:933] checking resume status, waiting at most 10 seconds\n",
      "I0310 06:47:49.199309 140666048968448 run_manager.py:951] resuming run from id: UnVuOnYxOmh4bHE0N3o0OnZpc3VhbGl6YXRpb24tZGVtbzpwZHdpdmVkaQ==\n",
      "I0310 06:47:49.237039 140666048968448 run_manager.py:963] upserting run before process can begin, waiting at most 10 seconds\n",
      "I0310 06:47:49.376586 140661941479168 run_manager.py:1048] saving patches\n",
      "I0310 06:47:49.378525 140661941479168 run_manager.py:1052] saving pip packages\n",
      "I0310 06:47:49.382028 140661941479168 run_manager.py:1054] initializing streaming files api\n",
      "I0310 06:47:49.385227 140661941479168 run_manager.py:1061] unblocking file change observer, beginning sync with W&B servers\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8d8c9b69286b4b91948544a4a5cd0e9c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Current iteration', max=191, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "Running loss: 0.698800"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 06:47:49.959861 140662613411584 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_104748-hxlq47z4/config.yaml\n",
      "I0310 06:47:50.076681 140662613411584 run_manager.py:677] file/dir created: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_104748-hxlq47z4/media/graph/graph_0_summary_f96f05d5.graph.json\n",
      "I0310 06:47:50.077244 140662613411584 run_manager.py:677] file/dir created: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_104748-hxlq47z4/wandb-metadata.json\n",
      "I0310 06:47:50.078046 140662613411584 run_manager.py:677] file/dir created: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_104748-hxlq47z4/wandb-events.jsonl\n",
      "I0310 06:47:50.078888 140662613411584 run_manager.py:677] file/dir created: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_104748-hxlq47z4/wandb-history.jsonl\n",
      "I0310 06:47:50.080479 140662613411584 run_manager.py:677] file/dir created: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_104748-hxlq47z4/requirements.txt\n",
      "I0310 06:47:50.081210 140662613411584 run_manager.py:677] file/dir created: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_104748-hxlq47z4/wandb-summary.json\n",
      "I0310 06:47:50.082091 140662613411584 run_manager.py:677] file/dir created: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_104748-hxlq47z4/media/graph\n",
      "I0310 06:47:50.082563 140662613411584 run_manager.py:677] file/dir created: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_104748-hxlq47z4/media\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.423495"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 06:48:05.965145 140662613411584 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_104748-hxlq47z4/wandb-metadata.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.637896"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 06:48:17.968822 140662613411584 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_104748-hxlq47z4/wandb-events.jsonl\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.527249"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 06:48:21.970003 140662613411584 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_104748-hxlq47z4/wandb-metadata.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.586445"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 06:48:27.038316 140662613411584 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_104748-hxlq47z4/wandb-summary.json\n",
      "I0310 06:48:27.052270 140662613411584 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_104748-hxlq47z4/wandb-history.jsonl\n",
      "I0310 06:48:32.940157 140666048968448 configuration_utils.py:118] Configuration saved in outputs/best_model/config.json\n",
      "I0310 06:48:33.171964 140662613411584 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_104748-hxlq47z4/wandb-summary.json\n",
      "I0310 06:48:33.178357 140662613411584 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_104748-hxlq47z4/wandb-history.jsonl\n",
      "I0310 06:48:33.467083 140666048968448 modeling_utils.py:298] Model weights saved in outputs/best_model/pytorch_model.bin\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.465774"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 06:48:37.172971 140662613411584 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_104748-hxlq47z4/wandb-metadata.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.403124"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 06:48:48.175947 140662613411584 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_104748-hxlq47z4/wandb-events.jsonl\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.487192"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 06:48:53.177196 140662613411584 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_104748-hxlq47z4/wandb-metadata.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.520999"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 06:49:05.627499 140666048968448 configuration_utils.py:118] Configuration saved in outputs/checkpoint-191-epoch-1/config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "Running loss: 0.301745"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 06:49:06.145545 140666048968448 modeling_utils.py:298] Model weights saved in outputs/checkpoint-191-epoch-1/pytorch_model.bin\n",
      "I0310 06:49:09.195023 140662613411584 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_104748-hxlq47z4/wandb-metadata.json\n",
      "I0310 06:49:12.419087 140666048968448 configuration_utils.py:118] Configuration saved in outputs/best_model/config.json\n",
      "I0310 06:49:12.938256 140666048968448 modeling_utils.py:298] Model weights saved in outputs/best_model/pytorch_model.bin\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "598652e918f74c92a56eff972f2bd525",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Current iteration', max=191, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.357526"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 06:49:17.197129 140662613411584 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_104748-hxlq47z4/wandb-summary.json\n",
      "I0310 06:49:17.199818 140662613411584 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_104748-hxlq47z4/wandb-history.jsonl\n",
      "I0310 06:49:18.197350 140662613411584 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_104748-hxlq47z4/wandb-events.jsonl\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.338364"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 06:49:23.198603 140662613411584 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_104748-hxlq47z4/wandb-summary.json\n",
      "I0310 06:49:23.199523 140662613411584 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_104748-hxlq47z4/wandb-history.jsonl\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.182843"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 06:49:25.199118 140662613411584 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_104748-hxlq47z4/wandb-metadata.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.310647"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 06:49:41.203074 140662613411584 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_104748-hxlq47z4/wandb-metadata.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.389972"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 06:49:48.205022 140662613411584 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_104748-hxlq47z4/wandb-events.jsonl\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.460994"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 06:49:57.207330 140662613411584 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_104748-hxlq47z4/wandb-metadata.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.303775"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 06:49:59.207922 140662613411584 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_104748-hxlq47z4/wandb-summary.json\n",
      "I0310 06:49:59.210373 140662613411584 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_104748-hxlq47z4/wandb-history.jsonl\n",
      "I0310 06:50:04.545636 140666048968448 configuration_utils.py:118] Configuration saved in outputs/best_model/config.json\n",
      "I0310 06:50:04.808708 140666048968448 modeling_utils.py:298] Model weights saved in outputs/best_model/pytorch_model.bin\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "Running loss: 0.176256"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 06:50:05.209357 140662613411584 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_104748-hxlq47z4/wandb-summary.json\n",
      "I0310 06:50:05.210268 140662613411584 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_104748-hxlq47z4/wandb-history.jsonl\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.207583"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 06:50:13.211642 140662613411584 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_104748-hxlq47z4/wandb-metadata.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.259083"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 06:50:19.213345 140662613411584 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_104748-hxlq47z4/wandb-events.jsonl\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.410513"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 06:50:30.216488 140662613411584 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_104748-hxlq47z4/wandb-metadata.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.240525"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 06:50:33.871136 140666048968448 configuration_utils.py:118] Configuration saved in outputs/checkpoint-382-epoch-2/config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "Running loss: 0.368610"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 06:50:34.118834 140666048968448 modeling_utils.py:298] Model weights saved in outputs/checkpoint-382-epoch-2/pytorch_model.bin\n",
      "I0310 06:50:40.094752 140666048968448 configuration_utils.py:118] Configuration saved in outputs/config.json\n",
      "I0310 06:50:40.601601 140666048968448 modeling_utils.py:298] Model weights saved in outputs/pytorch_model.bin\n",
      "I0310 06:50:40.615105 140666048968448 run_manager.py:1068] shutting down system stats and metadata service\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training of bert model complete. Saved to outputs/.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 06:50:41.254659 140666048968448 run_manager.py:1080] stopping streaming files and file change observer\n",
      "I0310 06:50:41.326479 140662613411584 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_104748-hxlq47z4/wandb-metadata.json\n",
      "I0310 06:50:41.330187 140666048968448 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_104748-hxlq47z4/wandb-events.jsonl\n"
     ]
    }
   ],
   "source": [
    "model_BERT.train_model(train_df_clean, eval_df=eval_df_clean)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### check model performance on validation data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-10T10:50:49.192436Z",
     "start_time": "2020-03-10T10:50:43.151011Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Converting to features started. Cache is not used.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "830833f72bbc4844968defd9277d9887",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1523), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e55e09071f4249e4afd4cc4dbc52b123",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=24), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'mcc': 0.5915974149823142, 'tp': 466, 'tn': 755, 'fp': 119, 'fn': 183, 'eval_loss': 0.45270544787247974, 'acc': 0.8017071569271176}\n"
     ]
    }
   ],
   "source": [
    "result, model_outputs, wrong_predictions = model_BERT.eval_model(eval_df_clean, acc=sklearn.metrics.accuracy_score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Roberta Model Training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-10T13:02:33.382479Z",
     "start_time": "2020-03-10T13:02:33.380712Z"
    }
   },
   "source": [
    "#### Setup the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-10T13:03:14.508803Z",
     "start_time": "2020-03-10T13:02:59.382233Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:02:59.605605 140666048968448 filelock.py:274] Lock 140662742471512 acquired on /home/priya/.cache/torch/transformers/e1a2a406b5a05063c31f4dfdee7608986ba7c6393f7f79db5e69dcd197208534.a7ab0e5de2d8321d6d6a15b199110f2c99be72976b7d151423cb8d8c261a13b6.lock\n",
      "I0310 09:02:59.606524 140666048968448 file_utils.py:479] https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-config.json not found in cache or force_download set to True, downloading to /home/priya/.cache/torch/transformers/tmp1fjnwz2k\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6e4d980035274593895499fb1b08b82c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Downloading', max=524, style=ProgressStyle(description_width=…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:02:59.855488 140666048968448 file_utils.py:489] storing https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-config.json in cache at /home/priya/.cache/torch/transformers/e1a2a406b5a05063c31f4dfdee7608986ba7c6393f7f79db5e69dcd197208534.a7ab0e5de2d8321d6d6a15b199110f2c99be72976b7d151423cb8d8c261a13b6\n",
      "I0310 09:02:59.856231 140666048968448 file_utils.py:492] creating metadata file for /home/priya/.cache/torch/transformers/e1a2a406b5a05063c31f4dfdee7608986ba7c6393f7f79db5e69dcd197208534.a7ab0e5de2d8321d6d6a15b199110f2c99be72976b7d151423cb8d8c261a13b6\n",
      "I0310 09:02:59.856696 140666048968448 filelock.py:318] Lock 140662742471512 released on /home/priya/.cache/torch/transformers/e1a2a406b5a05063c31f4dfdee7608986ba7c6393f7f79db5e69dcd197208534.a7ab0e5de2d8321d6d6a15b199110f2c99be72976b7d151423cb8d8c261a13b6.lock\n",
      "I0310 09:02:59.857347 140666048968448 configuration_utils.py:256] loading configuration file https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-config.json from cache at /home/priya/.cache/torch/transformers/e1a2a406b5a05063c31f4dfdee7608986ba7c6393f7f79db5e69dcd197208534.a7ab0e5de2d8321d6d6a15b199110f2c99be72976b7d151423cb8d8c261a13b6\n",
      "I0310 09:02:59.857834 140666048968448 configuration_utils.py:292] Model config RobertaConfig {\n",
      "  \"architectures\": [\n",
      "    \"RobertaForMaskedLM\"\n",
      "  ],\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"bos_token_id\": null,\n",
      "  \"do_sample\": false,\n",
      "  \"eos_token_ids\": null,\n",
      "  \"finetuning_task\": null,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"id2label\": {\n",
      "    \"0\": \"LABEL_0\",\n",
      "    \"1\": \"LABEL_1\"\n",
      "  },\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"is_decoder\": false,\n",
      "  \"label2id\": {\n",
      "    \"LABEL_0\": 0,\n",
      "    \"LABEL_1\": 1\n",
      "  },\n",
      "  \"layer_norm_eps\": 1e-05,\n",
      "  \"length_penalty\": 1.0,\n",
      "  \"max_length\": 20,\n",
      "  \"max_position_embeddings\": 514,\n",
      "  \"model_type\": \"roberta\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_beams\": 1,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"num_labels\": 2,\n",
      "  \"num_return_sequences\": 1,\n",
      "  \"output_attentions\": false,\n",
      "  \"output_hidden_states\": false,\n",
      "  \"output_past\": true,\n",
      "  \"pad_token_id\": null,\n",
      "  \"pruned_heads\": {},\n",
      "  \"repetition_penalty\": 1.0,\n",
      "  \"temperature\": 1.0,\n",
      "  \"top_k\": 50,\n",
      "  \"top_p\": 1.0,\n",
      "  \"torchscript\": false,\n",
      "  \"type_vocab_size\": 1,\n",
      "  \"use_bfloat16\": false,\n",
      "  \"vocab_size\": 50265\n",
      "}\n",
      "\n",
      "I0310 09:03:00.285185 140666048968448 filelock.py:274] Lock 140662731611776 acquired on /home/priya/.cache/torch/transformers/228756ed15b6d200d7cb45aaef08c087e2706f54cb912863d2efe07c89584eb7.49b88ba7ec2c26a7558dda98ca3884c3b80fa31cf43a1b1f23aef3ff81ba344e.lock\n",
      "I0310 09:03:00.288552 140666048968448 file_utils.py:479] https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-pytorch_model.bin not found in cache or force_download set to True, downloading to /home/priya/.cache/torch/transformers/tmpd5kikl8i\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "20a14ad9e1f749a4b340613ff99838a9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Downloading', max=501200538, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:03:09.971490 140666048968448 file_utils.py:489] storing https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-pytorch_model.bin in cache at /home/priya/.cache/torch/transformers/228756ed15b6d200d7cb45aaef08c087e2706f54cb912863d2efe07c89584eb7.49b88ba7ec2c26a7558dda98ca3884c3b80fa31cf43a1b1f23aef3ff81ba344e\n",
      "I0310 09:03:09.972035 140666048968448 file_utils.py:492] creating metadata file for /home/priya/.cache/torch/transformers/228756ed15b6d200d7cb45aaef08c087e2706f54cb912863d2efe07c89584eb7.49b88ba7ec2c26a7558dda98ca3884c3b80fa31cf43a1b1f23aef3ff81ba344e\n",
      "I0310 09:03:09.972424 140666048968448 filelock.py:318] Lock 140662731611776 released on /home/priya/.cache/torch/transformers/228756ed15b6d200d7cb45aaef08c087e2706f54cb912863d2efe07c89584eb7.49b88ba7ec2c26a7558dda98ca3884c3b80fa31cf43a1b1f23aef3ff81ba344e.lock\n",
      "I0310 09:03:09.972814 140666048968448 modeling_utils.py:461] loading weights file https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-pytorch_model.bin from cache at /home/priya/.cache/torch/transformers/228756ed15b6d200d7cb45aaef08c087e2706f54cb912863d2efe07c89584eb7.49b88ba7ec2c26a7558dda98ca3884c3b80fa31cf43a1b1f23aef3ff81ba344e\n",
      "I0310 09:03:12.791668 140666048968448 modeling_utils.py:546] Weights of RobertaForSequenceClassification not initialized from pretrained model: ['classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.out_proj.bias']\n",
      "I0310 09:03:12.792074 140666048968448 modeling_utils.py:552] Weights from pretrained model not used in RobertaForSequenceClassification: ['lm_head.bias', 'lm_head.dense.weight', 'lm_head.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight']\n",
      "I0310 09:03:12.994755 140666048968448 filelock.py:274] Lock 140662731609928 acquired on /home/priya/.cache/torch/transformers/d0c5776499adc1ded22493fae699da0971c1ee4c2587111707a4d177d20257a2.ef00af9e673c7160b4d41cfda1f48c5f4cba57d5142754525572a846a1ab1b9b.lock\n",
      "I0310 09:03:12.995350 140666048968448 file_utils.py:479] https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-vocab.json not found in cache or force_download set to True, downloading to /home/priya/.cache/torch/transformers/tmp3djdqfv_\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "245b04de06c241f6bd95da66b9901e43",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Downloading', max=898823, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:03:13.771920 140666048968448 file_utils.py:489] storing https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-vocab.json in cache at /home/priya/.cache/torch/transformers/d0c5776499adc1ded22493fae699da0971c1ee4c2587111707a4d177d20257a2.ef00af9e673c7160b4d41cfda1f48c5f4cba57d5142754525572a846a1ab1b9b\n",
      "I0310 09:03:13.774041 140666048968448 file_utils.py:492] creating metadata file for /home/priya/.cache/torch/transformers/d0c5776499adc1ded22493fae699da0971c1ee4c2587111707a4d177d20257a2.ef00af9e673c7160b4d41cfda1f48c5f4cba57d5142754525572a846a1ab1b9b\n",
      "I0310 09:03:13.776611 140666048968448 filelock.py:318] Lock 140662731609928 released on /home/priya/.cache/torch/transformers/d0c5776499adc1ded22493fae699da0971c1ee4c2587111707a4d177d20257a2.ef00af9e673c7160b4d41cfda1f48c5f4cba57d5142754525572a846a1ab1b9b.lock\n",
      "I0310 09:03:14.028954 140666048968448 filelock.py:274] Lock 140662731609928 acquired on /home/priya/.cache/torch/transformers/b35e7cd126cd4229a746b5d5c29a749e8e84438b14bcdb575950584fe33207e8.70bec105b4158ed9a1747fea67a43f5dee97855c64d62b6ec3742f4cfdb5feda.lock\n",
      "I0310 09:03:14.032194 140666048968448 file_utils.py:479] https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-merges.txt not found in cache or force_download set to True, downloading to /home/priya/.cache/torch/transformers/tmpgga3m_pc\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3be6fa595048414e9052cb6e14728073",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Downloading', max=456318, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:03:14.449236 140666048968448 file_utils.py:489] storing https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-merges.txt in cache at /home/priya/.cache/torch/transformers/b35e7cd126cd4229a746b5d5c29a749e8e84438b14bcdb575950584fe33207e8.70bec105b4158ed9a1747fea67a43f5dee97855c64d62b6ec3742f4cfdb5feda\n",
      "I0310 09:03:14.449947 140666048968448 file_utils.py:492] creating metadata file for /home/priya/.cache/torch/transformers/b35e7cd126cd4229a746b5d5c29a749e8e84438b14bcdb575950584fe33207e8.70bec105b4158ed9a1747fea67a43f5dee97855c64d62b6ec3742f4cfdb5feda\n",
      "I0310 09:03:14.450859 140666048968448 filelock.py:318] Lock 140662731609928 released on /home/priya/.cache/torch/transformers/b35e7cd126cd4229a746b5d5c29a749e8e84438b14bcdb575950584fe33207e8.70bec105b4158ed9a1747fea67a43f5dee97855c64d62b6ec3742f4cfdb5feda.lock\n",
      "I0310 09:03:14.451366 140666048968448 tokenization_utils.py:501] loading file https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-vocab.json from cache at /home/priya/.cache/torch/transformers/d0c5776499adc1ded22493fae699da0971c1ee4c2587111707a4d177d20257a2.ef00af9e673c7160b4d41cfda1f48c5f4cba57d5142754525572a846a1ab1b9b\n",
      "I0310 09:03:14.452030 140666048968448 tokenization_utils.py:501] loading file https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-merges.txt from cache at /home/priya/.cache/torch/transformers/b35e7cd126cd4229a746b5d5c29a749e8e84438b14bcdb575950584fe33207e8.70bec105b4158ed9a1747fea67a43f5dee97855c64d62b6ec3742f4cfdb5feda\n"
     ]
    }
   ],
   "source": [
    "model_Roberta = ClassificationModel('roberta', 'roberta-base', num_labels=2, use_cuda=True, cuda_device=0, args=train_args)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-10T13:06:32.170802Z",
     "start_time": "2020-03-10T13:03:32.010998Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Converting to features started. Cache is not used.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6128c09fdd5c4bb4854a6eecc6c9746f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=6090), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1fd28aef112a44c898a36366d3927835",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=2, style=ProgressStyle(description_width='initial…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "                Logging results to <a href=\"https://wandb.com\" target=\"_blank\">Weights & Biases</a> <a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">(Documentation)</a>.<br/>\n",
       "                Project page: <a href=\"https://app.wandb.ai/pdwivedi/visualization-demo\" target=\"_blank\">https://app.wandb.ai/pdwivedi/visualization-demo</a><br/>\n",
       "                Run page: <a href=\"https://app.wandb.ai/pdwivedi/visualization-demo/runs/1301xvu0\" target=\"_blank\">https://app.wandb.ai/pdwivedi/visualization-demo/runs/1301xvu0</a><br/>\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:03:33.974737 140666048968448 run_manager.py:924] system metrics and metadata threads started\n",
      "I0310 09:03:33.976883 140666048968448 run_manager.py:933] checking resume status, waiting at most 10 seconds\n",
      "I0310 09:03:34.157704 140666048968448 run_manager.py:951] resuming run from id: UnVuOnYxOjEzMDF4dnUwOnZpc3VhbGl6YXRpb24tZGVtbzpwZHdpdmVkaQ==\n",
      "I0310 09:03:34.181163 140666048968448 run_manager.py:963] upserting run before process can begin, waiting at most 10 seconds\n",
      "I0310 09:03:34.322329 140661933086464 run_manager.py:1048] saving patches\n",
      "I0310 09:03:34.324260 140661933086464 run_manager.py:1052] saving pip packages\n",
      "I0310 09:03:34.327748 140661933086464 run_manager.py:1054] initializing streaming files api\n",
      "I0310 09:03:34.331110 140661933086464 run_manager.py:1061] unblocking file change observer, beginning sync with W&B servers\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9979b7b8ddff410397a0c545561dff7f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Current iteration', max=191, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "Running loss: 0.754046"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:03:34.869752 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_130333-1301xvu0/config.yaml\n",
      "I0310 09:03:35.025761 140662192207616 run_manager.py:677] file/dir created: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_130333-1301xvu0/wandb-metadata.json\n",
      "I0310 09:03:35.026523 140662192207616 run_manager.py:677] file/dir created: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_130333-1301xvu0/wandb-history.jsonl\n",
      "I0310 09:03:35.027258 140662192207616 run_manager.py:677] file/dir created: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_130333-1301xvu0/wandb-events.jsonl\n",
      "I0310 09:03:35.027866 140662192207616 run_manager.py:677] file/dir created: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_130333-1301xvu0/wandb-summary.json\n",
      "I0310 09:03:35.028478 140662192207616 run_manager.py:677] file/dir created: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_130333-1301xvu0/media/graph/graph_0_summary_a68c9e4d.graph.json\n",
      "I0310 09:03:35.029096 140662192207616 run_manager.py:677] file/dir created: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_130333-1301xvu0/requirements.txt\n",
      "I0310 09:03:35.029689 140662192207616 run_manager.py:677] file/dir created: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_130333-1301xvu0/media/graph\n",
      "I0310 09:03:35.030164 140662192207616 run_manager.py:677] file/dir created: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_130333-1301xvu0/media\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.369981"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:03:50.873364 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_130333-1301xvu0/wandb-metadata.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.399023"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:04:02.876613 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_130333-1301xvu0/wandb-events.jsonl\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.410467"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:04:06.877681 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_130333-1301xvu0/wandb-metadata.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.525971"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:04:12.879320 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_130333-1301xvu0/wandb-summary.json\n",
      "I0310 09:04:12.880273 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_130333-1301xvu0/wandb-history.jsonl\n",
      "I0310 09:04:18.302709 140666048968448 configuration_utils.py:118] Configuration saved in outputs/best_model/config.json\n",
      "I0310 09:04:18.609557 140666048968448 modeling_utils.py:298] Model weights saved in outputs/best_model/pytorch_model.bin\n",
      "I0310 09:04:18.880850 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_130333-1301xvu0/wandb-summary.json\n",
      "I0310 09:04:18.881699 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_130333-1301xvu0/wandb-history.jsonl\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.408077"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:04:22.881886 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_130333-1301xvu0/wandb-metadata.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.573104"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:04:32.884463 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_130333-1301xvu0/wandb-events.jsonl\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.356302"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:04:38.886095 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_130333-1301xvu0/wandb-metadata.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.284750"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:04:51.961259 140666048968448 configuration_utils.py:118] Configuration saved in outputs/checkpoint-191-epoch-1/config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "Running loss: 0.370863"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:04:52.243735 140666048968448 modeling_utils.py:298] Model weights saved in outputs/checkpoint-191-epoch-1/pytorch_model.bin\n",
      "I0310 09:04:54.946515 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_130333-1301xvu0/wandb-metadata.json\n",
      "I0310 09:04:58.403270 140666048968448 configuration_utils.py:118] Configuration saved in outputs/best_model/config.json\n",
      "I0310 09:04:58.709264 140666048968448 modeling_utils.py:298] Model weights saved in outputs/best_model/pytorch_model.bin\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cc1913e43713456389a61842b7e0c821",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Current iteration', max=191, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.326816"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:05:02.950666 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_130333-1301xvu0/wandb-summary.json\n",
      "I0310 09:05:02.970115 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_130333-1301xvu0/wandb-events.jsonl\n",
      "I0310 09:05:02.977103 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_130333-1301xvu0/wandb-history.jsonl\n",
      "I0310 09:05:08.951627 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_130333-1301xvu0/wandb-summary.json\n",
      "I0310 09:05:08.952592 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_130333-1301xvu0/wandb-history.jsonl\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.262979"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:05:10.952171 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_130333-1301xvu0/wandb-metadata.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.271975"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:05:26.956253 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_130333-1301xvu0/wandb-metadata.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.275703"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:05:33.958224 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_130333-1301xvu0/wandb-events.jsonl\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.317784"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:05:42.960611 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_130333-1301xvu0/wandb-metadata.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.319234"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:05:45.965392 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_130333-1301xvu0/wandb-summary.json\n",
      "I0310 09:05:45.972975 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_130333-1301xvu0/wandb-history.jsonl\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "Running loss: 0.319138"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:05:51.965923 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_130333-1301xvu0/wandb-summary.json\n",
      "I0310 09:05:51.966868 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_130333-1301xvu0/wandb-history.jsonl\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.311203"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:05:58.968019 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_130333-1301xvu0/wandb-metadata.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.386319"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:06:03.969316 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_130333-1301xvu0/wandb-events.jsonl\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.301595"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:06:14.972288 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_130333-1301xvu0/wandb-metadata.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.291711"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:06:21.698886 140666048968448 configuration_utils.py:118] Configuration saved in outputs/checkpoint-382-epoch-2/config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "Running loss: 0.091935"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:06:21.996402 140666048968448 modeling_utils.py:298] Model weights saved in outputs/checkpoint-382-epoch-2/pytorch_model.bin\n",
      "I0310 09:06:28.293530 140666048968448 configuration_utils.py:118] Configuration saved in outputs/config.json\n",
      "I0310 09:06:28.591836 140666048968448 modeling_utils.py:298] Model weights saved in outputs/pytorch_model.bin\n",
      "I0310 09:06:28.687106 140666048968448 run_manager.py:1068] shutting down system stats and metadata service\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training of roberta model complete. Saved to outputs/.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:06:28.976661 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_130333-1301xvu0/wandb-events.jsonl\n",
      "I0310 09:06:29.238617 140666048968448 run_manager.py:1080] stopping streaming files and file change observer\n",
      "I0310 09:06:29.977487 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_130333-1301xvu0/wandb-metadata.json\n"
     ]
    }
   ],
   "source": [
    "model_Roberta.train_model(train_df_clean, eval_df=eval_df_clean)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Evaluate the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-10T13:06:55.993060Z",
     "start_time": "2020-03-10T13:06:49.926986Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Converting to features started. Cache is not used.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "08ee45f577124d54a9c79fd9983825ae",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1523), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a13eb45db0504da8bb1e8f7527274b00",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=24), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'mcc': 0.6223595391558661, 'tp': 459, 'tn': 784, 'fp': 90, 'fn': 190, 'eval_loss': 0.45775141566991806, 'acc': 0.8161523309258043}\n"
     ]
    }
   ],
   "source": [
    "result, model_outputs, wrong_predictions = model_Roberta.eval_model(eval_df_clean, acc=sklearn.metrics.accuracy_score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ALBERT Model training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-10T13:10:26.361730Z",
     "start_time": "2020-03-10T13:10:21.727984Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:10:21.979554 140666048968448 filelock.py:274] Lock 140662731518696 acquired on /home/priya/.cache/torch/transformers/0bbb1531ce82f042a813219ffeed7a1fa1f44cd8f78a652c47fc5311e0d40231.49ede2f5cbd21a453ab03ed1214f9068f024910f34b5023577f3d0068326f7b0.lock\n",
      "I0310 09:10:21.983073 140666048968448 file_utils.py:479] https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-v2-config.json not found in cache or force_download set to True, downloading to /home/priya/.cache/torch/transformers/tmpcxjmcovj\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a0e58e6912294e73b059b249a7ea6bd8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Downloading', max=534, style=ProgressStyle(description_width=…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:10:22.234151 140666048968448 file_utils.py:489] storing https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-v2-config.json in cache at /home/priya/.cache/torch/transformers/0bbb1531ce82f042a813219ffeed7a1fa1f44cd8f78a652c47fc5311e0d40231.49ede2f5cbd21a453ab03ed1214f9068f024910f34b5023577f3d0068326f7b0\n",
      "I0310 09:10:22.234591 140666048968448 file_utils.py:492] creating metadata file for /home/priya/.cache/torch/transformers/0bbb1531ce82f042a813219ffeed7a1fa1f44cd8f78a652c47fc5311e0d40231.49ede2f5cbd21a453ab03ed1214f9068f024910f34b5023577f3d0068326f7b0\n",
      "I0310 09:10:22.235194 140666048968448 filelock.py:318] Lock 140662731518696 released on /home/priya/.cache/torch/transformers/0bbb1531ce82f042a813219ffeed7a1fa1f44cd8f78a652c47fc5311e0d40231.49ede2f5cbd21a453ab03ed1214f9068f024910f34b5023577f3d0068326f7b0.lock\n",
      "I0310 09:10:22.235548 140666048968448 configuration_utils.py:256] loading configuration file https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-v2-config.json from cache at /home/priya/.cache/torch/transformers/0bbb1531ce82f042a813219ffeed7a1fa1f44cd8f78a652c47fc5311e0d40231.49ede2f5cbd21a453ab03ed1214f9068f024910f34b5023577f3d0068326f7b0\n",
      "I0310 09:10:22.236091 140666048968448 configuration_utils.py:292] Model config AlbertConfig {\n",
      "  \"architectures\": [\n",
      "    \"AlbertForMaskedLM\"\n",
      "  ],\n",
      "  \"attention_probs_dropout_prob\": 0,\n",
      "  \"bos_token_id\": null,\n",
      "  \"classifier_dropout_prob\": 0.1,\n",
      "  \"do_sample\": false,\n",
      "  \"down_scale_factor\": 1,\n",
      "  \"embedding_size\": 128,\n",
      "  \"eos_token_ids\": null,\n",
      "  \"finetuning_task\": null,\n",
      "  \"gap_size\": 0,\n",
      "  \"hidden_act\": \"gelu_new\",\n",
      "  \"hidden_dropout_prob\": 0,\n",
      "  \"hidden_size\": 768,\n",
      "  \"id2label\": {\n",
      "    \"0\": \"LABEL_0\",\n",
      "    \"1\": \"LABEL_1\"\n",
      "  },\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"inner_group_num\": 1,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"is_decoder\": false,\n",
      "  \"label2id\": {\n",
      "    \"LABEL_0\": 0,\n",
      "    \"LABEL_1\": 1\n",
      "  },\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"length_penalty\": 1.0,\n",
      "  \"max_length\": 20,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"model_type\": \"albert\",\n",
      "  \"net_structure_type\": 0,\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_beams\": 1,\n",
      "  \"num_hidden_groups\": 1,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"num_labels\": 2,\n",
      "  \"num_memory_blocks\": 0,\n",
      "  \"num_return_sequences\": 1,\n",
      "  \"output_attentions\": false,\n",
      "  \"output_hidden_states\": false,\n",
      "  \"output_past\": true,\n",
      "  \"pad_token_id\": null,\n",
      "  \"pruned_heads\": {},\n",
      "  \"repetition_penalty\": 1.0,\n",
      "  \"temperature\": 1.0,\n",
      "  \"top_k\": 50,\n",
      "  \"top_p\": 1.0,\n",
      "  \"torchscript\": false,\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"use_bfloat16\": false,\n",
      "  \"vocab_size\": 30000\n",
      "}\n",
      "\n",
      "I0310 09:10:22.483689 140666048968448 filelock.py:274] Lock 140662629536600 acquired on /home/priya/.cache/torch/transformers/a175de1d3c60bba6e74bd034c02a34d909d9f36a0cf472b02301c8790ba44834.ab806923413c2af99835e13fdbb6014b24af86b0de8edc2d71ef5c646fc54f24.lock\n",
      "I0310 09:10:22.486903 140666048968448 file_utils.py:479] https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-v2-pytorch_model.bin not found in cache or force_download set to True, downloading to /home/priya/.cache/torch/transformers/tmp0i8jifdi\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1d6bf774970e4bd5b0a538d840c2829e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Downloading', max=47376696, style=ProgressStyle(description_w…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:10:25.050044 140666048968448 file_utils.py:489] storing https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-v2-pytorch_model.bin in cache at /home/priya/.cache/torch/transformers/a175de1d3c60bba6e74bd034c02a34d909d9f36a0cf472b02301c8790ba44834.ab806923413c2af99835e13fdbb6014b24af86b0de8edc2d71ef5c646fc54f24\n",
      "I0310 09:10:25.050722 140666048968448 file_utils.py:492] creating metadata file for /home/priya/.cache/torch/transformers/a175de1d3c60bba6e74bd034c02a34d909d9f36a0cf472b02301c8790ba44834.ab806923413c2af99835e13fdbb6014b24af86b0de8edc2d71ef5c646fc54f24\n",
      "I0310 09:10:25.051204 140666048968448 filelock.py:318] Lock 140662629536600 released on /home/priya/.cache/torch/transformers/a175de1d3c60bba6e74bd034c02a34d909d9f36a0cf472b02301c8790ba44834.ab806923413c2af99835e13fdbb6014b24af86b0de8edc2d71ef5c646fc54f24.lock\n",
      "I0310 09:10:25.051642 140666048968448 modeling_utils.py:461] loading weights file https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-v2-pytorch_model.bin from cache at /home/priya/.cache/torch/transformers/a175de1d3c60bba6e74bd034c02a34d909d9f36a0cf472b02301c8790ba44834.ab806923413c2af99835e13fdbb6014b24af86b0de8edc2d71ef5c646fc54f24\n",
      "I0310 09:10:25.419910 140666048968448 modeling_utils.py:546] Weights of AlbertForSequenceClassification not initialized from pretrained model: ['classifier.weight', 'classifier.bias']\n",
      "I0310 09:10:25.420291 140666048968448 modeling_utils.py:552] Weights from pretrained model not used in AlbertForSequenceClassification: ['predictions.bias', 'predictions.LayerNorm.weight', 'predictions.LayerNorm.bias', 'predictions.dense.weight', 'predictions.dense.bias', 'predictions.decoder.weight', 'predictions.decoder.bias']\n",
      "I0310 09:10:25.682727 140666048968448 filelock.py:274] Lock 140662629535816 acquired on /home/priya/.cache/torch/transformers/dd1588b85b6fdce1320e224d29ad062e97588e17326b9d05a0b29ee84b8f5f93.c81d4deb77aec08ce575b7a39a989a79dd54f321bfb82c2b54dd35f52f8182cf.lock\n",
      "I0310 09:10:25.683535 140666048968448 file_utils.py:479] https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-v2-spiece.model not found in cache or force_download set to True, downloading to /home/priya/.cache/torch/transformers/tmpwny4g6h7\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "490e3b9f498c4a73967c5d3712e5b744",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Downloading', max=760289, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:10:26.311834 140666048968448 file_utils.py:489] storing https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-v2-spiece.model in cache at /home/priya/.cache/torch/transformers/dd1588b85b6fdce1320e224d29ad062e97588e17326b9d05a0b29ee84b8f5f93.c81d4deb77aec08ce575b7a39a989a79dd54f321bfb82c2b54dd35f52f8182cf\n",
      "I0310 09:10:26.312609 140666048968448 file_utils.py:492] creating metadata file for /home/priya/.cache/torch/transformers/dd1588b85b6fdce1320e224d29ad062e97588e17326b9d05a0b29ee84b8f5f93.c81d4deb77aec08ce575b7a39a989a79dd54f321bfb82c2b54dd35f52f8182cf\n",
      "I0310 09:10:26.313185 140666048968448 filelock.py:318] Lock 140662629535816 released on /home/priya/.cache/torch/transformers/dd1588b85b6fdce1320e224d29ad062e97588e17326b9d05a0b29ee84b8f5f93.c81d4deb77aec08ce575b7a39a989a79dd54f321bfb82c2b54dd35f52f8182cf.lock\n",
      "I0310 09:10:26.313593 140666048968448 tokenization_utils.py:501] loading file https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-v2-spiece.model from cache at /home/priya/.cache/torch/transformers/dd1588b85b6fdce1320e224d29ad062e97588e17326b9d05a0b29ee84b8f5f93.c81d4deb77aec08ce575b7a39a989a79dd54f321bfb82c2b54dd35f52f8182cf\n"
     ]
    }
   ],
   "source": [
    "model_albert = ClassificationModel('albert', 'albert-base-v2', num_labels=2, use_cuda=True, cuda_device=0, args=train_args)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-10T13:13:33.646215Z",
     "start_time": "2020-03-10T13:10:44.375847Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Converting to features started. Cache is not used.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "59567763a8c3474b8bae7ecc41abb029",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=6090), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "33158e3938e7452e9941e98a8f6c90f0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=2, style=ProgressStyle(description_width='initial…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "                Logging results to <a href=\"https://wandb.com\" target=\"_blank\">Weights & Biases</a> <a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">(Documentation)</a>.<br/>\n",
       "                Project page: <a href=\"https://app.wandb.ai/pdwivedi/visualization-demo\" target=\"_blank\">https://app.wandb.ai/pdwivedi/visualization-demo</a><br/>\n",
       "                Run page: <a href=\"https://app.wandb.ai/pdwivedi/visualization-demo/runs/80ujc6ed\" target=\"_blank\">https://app.wandb.ai/pdwivedi/visualization-demo/runs/80ujc6ed</a><br/>\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:10:46.049408 140666048968448 run_manager.py:924] system metrics and metadata threads started\n",
      "I0310 09:10:46.051529 140666048968448 run_manager.py:933] checking resume status, waiting at most 10 seconds\n",
      "I0310 09:10:46.279228 140666048968448 run_manager.py:951] resuming run from id: UnVuOnYxOjgwdWpjNmVkOnZpc3VhbGl6YXRpb24tZGVtbzpwZHdpdmVkaQ==\n",
      "I0310 09:10:46.318503 140666048968448 run_manager.py:963] upserting run before process can begin, waiting at most 10 seconds\n",
      "I0310 09:10:46.540079 140661856663296 run_manager.py:1048] saving patches\n",
      "I0310 09:10:46.541958 140661856663296 run_manager.py:1052] saving pip packages\n",
      "I0310 09:10:46.545370 140661856663296 run_manager.py:1054] initializing streaming files api\n",
      "I0310 09:10:46.548172 140661856663296 run_manager.py:1061] unblocking file change observer, beginning sync with W&B servers\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2eac4284967348b7ae9f7a48df195881",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Current iteration', max=191, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:10:46.864837 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_131045-80ujc6ed/config.yaml\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "Running loss: 0.685512"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:10:47.020050 140662192207616 run_manager.py:677] file/dir created: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_131045-80ujc6ed/wandb-history.jsonl\n",
      "I0310 09:10:47.020910 140662192207616 run_manager.py:677] file/dir created: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_131045-80ujc6ed/wandb-events.jsonl\n",
      "I0310 09:10:47.022298 140662192207616 run_manager.py:677] file/dir created: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_131045-80ujc6ed/requirements.txt\n",
      "I0310 09:10:47.022899 140662192207616 run_manager.py:677] file/dir created: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_131045-80ujc6ed/media/graph/graph_0_summary_1a989aa0.graph.json\n",
      "I0310 09:10:47.023462 140662192207616 run_manager.py:677] file/dir created: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_131045-80ujc6ed/wandb-summary.json\n",
      "I0310 09:10:47.024189 140662192207616 run_manager.py:677] file/dir created: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_131045-80ujc6ed/wandb-metadata.json\n",
      "I0310 09:10:47.024765 140662192207616 run_manager.py:677] file/dir created: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_131045-80ujc6ed/media\n",
      "I0310 09:10:47.025144 140662192207616 run_manager.py:677] file/dir created: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_131045-80ujc6ed/media/graph\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.630319"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:11:02.868142 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_131045-80ujc6ed/wandb-metadata.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.762322"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:11:14.872592 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_131045-80ujc6ed/wandb-events.jsonl\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.639655"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:11:18.873706 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_131045-80ujc6ed/wandb-metadata.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.534089"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:11:21.874924 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_131045-80ujc6ed/wandb-history.jsonl\n",
      "I0310 09:11:21.876337 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_131045-80ujc6ed/wandb-summary.json\n",
      "I0310 09:11:28.121426 140666048968448 configuration_utils.py:118] Configuration saved in outputs/best_model/config.json\n",
      "I0310 09:11:28.187113 140666048968448 modeling_utils.py:298] Model weights saved in outputs/best_model/pytorch_model.bin\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.640330"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:11:28.876506 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_131045-80ujc6ed/wandb-history.jsonl\n",
      "I0310 09:11:28.877196 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_131045-80ujc6ed/wandb-summary.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.572716"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:11:34.878118 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_131045-80ujc6ed/wandb-metadata.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.528466"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:11:44.880729 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_131045-80ujc6ed/wandb-events.jsonl\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.667771"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:11:50.882390 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_131045-80ujc6ed/wandb-metadata.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.533343"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:11:59.244931 140666048968448 configuration_utils.py:118] Configuration saved in outputs/checkpoint-191-epoch-1/config.json\n",
      "I0310 09:11:59.305748 140666048968448 modeling_utils.py:298] Model weights saved in outputs/checkpoint-191-epoch-1/pytorch_model.bin\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "Running loss: 0.479151"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:12:05.581230 140666048968448 configuration_utils.py:118] Configuration saved in outputs/best_model/config.json\n",
      "I0310 09:12:05.614296 140666048968448 modeling_utils.py:298] Model weights saved in outputs/best_model/pytorch_model.bin\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d2ce14a50d534daeb72b90480926a3e8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Current iteration', max=191, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.468530"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:12:06.886604 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_131045-80ujc6ed/wandb-metadata.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.490144"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:12:08.887451 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_131045-80ujc6ed/wandb-history.jsonl\n",
      "I0310 09:12:08.889034 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_131045-80ujc6ed/wandb-summary.json\n",
      "I0310 09:12:14.889071 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_131045-80ujc6ed/wandb-events.jsonl\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.407895"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:12:15.889326 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_131045-80ujc6ed/wandb-history.jsonl\n",
      "I0310 09:12:15.889835 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_131045-80ujc6ed/wandb-summary.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.386610"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:12:22.891403 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_131045-80ujc6ed/wandb-metadata.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.558779"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:12:38.895255 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_131045-80ujc6ed/wandb-metadata.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.581347"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:12:44.896793 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_131045-80ujc6ed/wandb-events.jsonl\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.494742"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:12:49.898380 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_131045-80ujc6ed/wandb-history.jsonl\n",
      "I0310 09:12:49.899195 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_131045-80ujc6ed/wandb-summary.json\n",
      "I0310 09:12:54.899547 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_131045-80ujc6ed/wandb-metadata.json\n",
      "I0310 09:12:55.652584 140666048968448 configuration_utils.py:118] Configuration saved in outputs/best_model/config.json\n",
      "I0310 09:12:55.684015 140666048968448 modeling_utils.py:298] Model weights saved in outputs/best_model/pytorch_model.bin\n",
      "I0310 09:12:55.899852 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_131045-80ujc6ed/wandb-history.jsonl\n",
      "I0310 09:12:55.900697 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_131045-80ujc6ed/wandb-summary.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.468477"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:13:10.905192 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_131045-80ujc6ed/wandb-metadata.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.547322"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:13:15.906450 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_131045-80ujc6ed/wandb-events.jsonl\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running loss: 0.492901"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:13:23.634442 140666048968448 configuration_utils.py:118] Configuration saved in outputs/checkpoint-382-epoch-2/config.json\n",
      "I0310 09:13:23.694975 140666048968448 modeling_utils.py:298] Model weights saved in outputs/checkpoint-382-epoch-2/pytorch_model.bin\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "Running loss: 0.522778"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:13:26.909488 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_131045-80ujc6ed/wandb-metadata.json\n",
      "I0310 09:13:30.022613 140666048968448 configuration_utils.py:118] Configuration saved in outputs/best_model/config.json\n",
      "I0310 09:13:30.060360 140666048968448 modeling_utils.py:298] Model weights saved in outputs/best_model/pytorch_model.bin\n",
      "I0310 09:13:30.064007 140666048968448 configuration_utils.py:118] Configuration saved in outputs/config.json\n",
      "I0310 09:13:30.125827 140666048968448 modeling_utils.py:298] Model weights saved in outputs/pytorch_model.bin\n",
      "I0310 09:13:30.128537 140666048968448 run_manager.py:1068] shutting down system stats and metadata service\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training of albert model complete. Saved to outputs/.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0310 09:13:30.911246 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_131045-80ujc6ed/wandb-events.jsonl\n",
      "I0310 09:13:31.232758 140666048968448 run_manager.py:1080] stopping streaming files and file change observer\n",
      "I0310 09:13:31.912103 140662192207616 run_manager.py:688] file/dir modified: /home/priya/Documents/AI_Apps/simpletransformers_blog/wandb/run-20200310_131045-80ujc6ed/wandb-metadata.json\n"
     ]
    }
   ],
   "source": [
    "model_albert.train_model(train_df_clean, eval_df=eval_df_clean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-10T13:13:50.788829Z",
     "start_time": "2020-03-10T13:13:44.158174Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Converting to features started. Cache is not used.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "67fe5ee2d3ba479b8593db5a2e5c10f6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1523), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c58b2100c20d4d58a957b015518fd6a7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=24), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'mcc': 0.5121674642936882, 'tp': 433, 'tn': 730, 'fp': 144, 'fn': 216, 'eval_loss': 0.5123823285102844, 'acc': 0.7636244254760342}\n"
     ]
    }
   ],
   "source": [
    "result, model_outputs, wrong_predictions = model_albert.eval_model(eval_df_clean, acc=sklearn.metrics.accuracy_score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Perform prediction - Test Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-10T13:15:33.406161Z",
     "start_time": "2020-03-10T13:15:33.397448Z"
    }
   },
   "outputs": [],
   "source": [
    "test_df = pd.read_csv(\"./test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-10T13:15:35.878541Z",
     "start_time": "2020-03-10T13:15:35.401785Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>happened terrible crash</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Heard about earthquake different cities everyone</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>there forest geese fleeing across street cannot</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9</td>\n",
       "      <td>Apocalypse lighting Spokane wildfires</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11</td>\n",
       "      <td>Typhoon Soudelor kills China Taiwan</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id                                              text\n",
       "0   0                           happened terrible crash\n",
       "1   2  Heard about earthquake different cities everyone\n",
       "2   3   there forest geese fleeing across street cannot\n",
       "3   9             Apocalypse lighting Spokane wildfires\n",
       "4  11               Typhoon Soudelor kills China Taiwan"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Drop columns for keyword and location\n",
    "columns_todrop = ['keyword','location']\n",
    "test_df.drop(columns=columns_todrop,inplace=True)\n",
    "### Change contractions\n",
    "test_df['text']=test_df['text'].apply(remove_contractions)\n",
    "## Clean Data set\n",
    "test_df['text'] =test_df['text'].apply(clean_dataset)\n",
    "test_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-10T13:16:16.452764Z",
     "start_time": "2020-03-10T13:16:04.187320Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Converting to features started. Cache is not used.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d12153f84be34ffb81dce0ed82acd5bb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3263), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a490c7e7ce9f4ce0945e06c57b19f05c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=51), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "predictions, raw_outputs = model_Roberta.predict(test_df['text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-10T13:16:59.088677Z",
     "start_time": "2020-03-10T13:16:59.083176Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3258</th>\n",
       "      <td>10861</td>\n",
       "      <td>EARTHQUAKE SAFETY ANGELES SAFETY FASTENERS</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3259</th>\n",
       "      <td>10865</td>\n",
       "      <td>Storm worse hurricane city3others hardest look...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3260</th>\n",
       "      <td>10868</td>\n",
       "      <td>Green derailment Chicago</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3261</th>\n",
       "      <td>10874</td>\n",
       "      <td>issues Hazardous Weather Outlook</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3262</th>\n",
       "      <td>10875</td>\n",
       "      <td>CityofCalgary activated Municipal Emergency yy...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         id                                               text  target\n",
       "3258  10861         EARTHQUAKE SAFETY ANGELES SAFETY FASTENERS       0\n",
       "3259  10865  Storm worse hurricane city3others hardest look...       1\n",
       "3260  10868                           Green derailment Chicago       1\n",
       "3261  10874                   issues Hazardous Weather Outlook       1\n",
       "3262  10875  CityofCalgary activated Municipal Emergency yy...       1"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df['target']=predictions\n",
    "test_df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-10T13:16:49.612020Z",
     "start_time": "2020-03-10T13:16:49.607660Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    2132\n",
       "1    1131\n",
       "Name: target, dtype: int64"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df['target'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Perform predictions on random tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-10T13:24:54.783468Z",
     "start_time": "2020-03-10T13:24:54.778026Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'COVID19 spread across coming weeks focus limiting epidemic preserving'"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_tweet1 = \"#COVID19 will spread across U.S. in coming weeks. We’ll get past it, but must focus on limiting the epidemic, and preserving lif\"\n",
    "test_tweet1 = remove_contractions(test_tweet1)\n",
    "test_tweet1 = clean_dataset(test_tweet1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-10T13:25:41.711325Z",
     "start_time": "2020-03-10T13:25:41.306161Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Converting to features started. Cache is not used.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "054c4086d79343acab6ac1e9c34d3f2d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c70654d113774f67a670f0d63f4993dc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prediction is:  Real\n"
     ]
    }
   ],
   "source": [
    "predictions, _ = model_Roberta.predict([test_tweet1])\n",
    "response_dict = {0: 'Fake', 1: 'Real'}\n",
    "print(\"Prediction is: \", response_dict[predictions[0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-10T13:27:28.037706Z",
     "start_time": "2020-03-10T13:27:28.034760Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'BREAKING Confirmed flooding trading floor flooded under water'"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_tweet2 = \"BREAKING: Confirmed flooding on NYSE. The trading floor is flooded under more than 3 feet of water.\"\n",
    "test_tweet2 = remove_contractions(test_tweet2)\n",
    "test_tweet2 = clean_dataset(test_tweet2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-10T13:27:37.505779Z",
     "start_time": "2020-03-10T13:27:37.144575Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Converting to features started. Cache is not used.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e973829c1a354a9c9f3a166b03bfc184",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a1809d7f8e5f4931bc7f3d0b2b37f6ba",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prediction is:  Real\n"
     ]
    }
   ],
   "source": [
    "predictions, _ = model_Roberta.predict([test_tweet2])\n",
    "response_dict = {0: 'Fake', 1: 'Real'}\n",
    "print(\"Prediction is: \", response_dict[predictions[0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-10T13:30:33.172857Z",
     "start_time": "2020-03-10T13:30:33.169914Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Everything ABLAZE Please'"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_tweet3 = \"Everything is ABLAZE. Please run!!\"\n",
    "test_tweet3 = remove_contractions(test_tweet3)\n",
    "test_tweet3 = clean_dataset(test_tweet3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-10T13:30:36.187689Z",
     "start_time": "2020-03-10T13:30:35.937815Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Converting to features started. Cache is not used.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "daa4edf2031b4c7b8069c0977759b37e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a4c126e5a13b4c2baf2517d7cdb3763c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prediction is:  Fake\n"
     ]
    }
   ],
   "source": [
    "predictions, _ = model_Roberta.predict([test_tweet3])\n",
    "response_dict = {0: 'Fake', 1: 'Real'}\n",
    "print(\"Prediction is: \", response_dict[predictions[0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:tf_torch] *",
   "language": "python",
   "name": "conda-env-tf_torch-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}