rating_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import collections\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from argparse import Namespace\n",
    "import pandas as pd\n",
    "import re\n",
    "import torch\n",
    "import torchtext\n",
    "import torch.nn as nn\n",
    "from pathlib import Path\n",
    "from gensim.models import Word2Vec\n",
    "import time\n",
    "import copy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Plan A: Self-trained Word2Vec + Homemade LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set Numpy and PyTorch seeds\n",
    "def set_seeds(seed, cuda):\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    if cuda:\n",
    "        torch.cuda.manual_seed_all(seed)\n",
    "        \n",
    "# Creating directories\n",
    "def create_dirs(dirpath):\n",
    "    if not os.path.exists(dirpath):\n",
    "        os.makedirs(dirpath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using CUDA: True\n"
     ]
    }
   ],
   "source": [
    "# Arguments\n",
    "args = Namespace(\n",
    "    seed=1234,\n",
    "    cuda=True,\n",
    "    path=\"data\",\n",
    "    w2vmodel_path=\"language.w2v.model\",\n",
    "    batch_size=16,\n",
    "    num_workers=4\n",
    ")\n",
    "# Set seeds\n",
    "set_seeds(seed=args.seed, cuda=args.cuda)\n",
    "\n",
    "# Check CUDA\n",
    "if not torch.cuda.is_available():\n",
    "    args.cuda = False\n",
    "args.device = torch.device(\"cuda\" if args.cuda else \"cpu\")\n",
    "print(\"Using CUDA: {}\".format(args.cuda))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preparing the data\n",
    "\n",
    "Use panda to conveniently import three `TSV` data files as `DataFrame`s."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "path =  Path(args.path)\n",
    "training_df = pd.read_csv(path / 'training_set.ss', sep='\\t')\n",
    "test_df = pd.read_csv(path / 'test_set.ss', sep='\\t')\n",
    "validation_df = pd.read_csv(path / 'validation_set.ss', sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>product_id</th>\n",
       "      <th>rating</th>\n",
       "      <th>review_content</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ur0116181/</td>\n",
       "      <td>\\tt0185937</td>\n",
       "      <td>1</td>\n",
       "      <td>sometimes popular opinion really sucks about a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ur0116181/</td>\n",
       "      <td>\\tt0169547</td>\n",
       "      <td>10</td>\n",
       "      <td>this is an amazing piece of work and it probab...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ur0116181/</td>\n",
       "      <td>\\tt0478304</td>\n",
       "      <td>5</td>\n",
       "      <td>this movie does not belong in a cinema . &lt;ssss...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ur0116181/</td>\n",
       "      <td>\\tt0195685</td>\n",
       "      <td>10</td>\n",
       "      <td>the first nominees for next year 's oscars in ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ur0116181/</td>\n",
       "      <td>\\tt0217869</td>\n",
       "      <td>10</td>\n",
       "      <td>my expectations for writer/director m. night s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>ur0116181/</td>\n",
       "      <td>\\tt0181689</td>\n",
       "      <td>10</td>\n",
       "      <td>let me warn you that a.i. was my favorite film...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>ur0116181/</td>\n",
       "      <td>\\tt0062622</td>\n",
       "      <td>10</td>\n",
       "      <td>this was the movie that made me fall in love w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>ur0116181/</td>\n",
       "      <td>\\tt0155267</td>\n",
       "      <td>9</td>\n",
       "      <td>i do n't want to jinx this movie , but this is...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>ur0116181/</td>\n",
       "      <td>\\tt0140352</td>\n",
       "      <td>10</td>\n",
       "      <td>this film is unavoidably being compared to dir...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>ur0116181/</td>\n",
       "      <td>\\tt0887912</td>\n",
       "      <td>10</td>\n",
       "      <td>without a doubt this is the best war film sinc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>ur0116181/</td>\n",
       "      <td>\\tt0372784</td>\n",
       "      <td>10</td>\n",
       "      <td>having seen this film a 2nd time only confirms...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>ur0116181/</td>\n",
       "      <td>\\tt1285016</td>\n",
       "      <td>10</td>\n",
       "      <td>like others before me when i initially heard a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>ur0116181/</td>\n",
       "      <td>\\tt0167404</td>\n",
       "      <td>10</td>\n",
       "      <td>the word of mouth -lrb- wom -rrb- on this movi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>ur0116181/</td>\n",
       "      <td>\\tt1152836</td>\n",
       "      <td>7</td>\n",
       "      <td>maybe this movie suffers from comparisons with...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>ur0116181/</td>\n",
       "      <td>\\tt0257044</td>\n",
       "      <td>9</td>\n",
       "      <td>sam mendes 's directorial follow up to `` amer...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>ur0116181/</td>\n",
       "      <td>\\tt0416449</td>\n",
       "      <td>8</td>\n",
       "      <td>this is n't the godfather or close encounters ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>ur0116181/</td>\n",
       "      <td>\\tt1568346</td>\n",
       "      <td>9</td>\n",
       "      <td>the novel &amp; the swedish movie are not great by...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>ur0116181/</td>\n",
       "      <td>\\tt0085809</td>\n",
       "      <td>10</td>\n",
       "      <td>i put this film in the same league as `` 2001 ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>ur0550732/</td>\n",
       "      <td>\\tt0478311</td>\n",
       "      <td>10</td>\n",
       "      <td>we start all of our reviews with the following...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>ur0550732/</td>\n",
       "      <td>\\tt0467200</td>\n",
       "      <td>9</td>\n",
       "      <td>we start all of our reviews with the following...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>ur0550732/</td>\n",
       "      <td>\\tt0829482</td>\n",
       "      <td>10</td>\n",
       "      <td>we start all of our reviews with the following...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>ur0550732/</td>\n",
       "      <td>\\tt0328107</td>\n",
       "      <td>8</td>\n",
       "      <td>denzel washington and dakota fanning both dese...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>ur0550732/</td>\n",
       "      <td>\\tt0343660</td>\n",
       "      <td>9</td>\n",
       "      <td>these two have it going . &lt;sssss&gt; in this thei...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>ur0550732/</td>\n",
       "      <td>\\tt0473308</td>\n",
       "      <td>8</td>\n",
       "      <td>we start all of our reviews with the following...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>ur0550732/</td>\n",
       "      <td>\\tt0402894</td>\n",
       "      <td>8</td>\n",
       "      <td>we start all of our reviews with the following...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>ur0550732/</td>\n",
       "      <td>\\tt0203755</td>\n",
       "      <td>9</td>\n",
       "      <td>she is one of the sexiest actresses in the mov...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>ur0550732/</td>\n",
       "      <td>\\tt0208874</td>\n",
       "      <td>9</td>\n",
       "      <td>what a wonderful political movie . &lt;sssss&gt; it ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>ur0550732/</td>\n",
       "      <td>\\tt0223897</td>\n",
       "      <td>10</td>\n",
       "      <td>we loved everything about this movie , from th...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>ur0550732/</td>\n",
       "      <td>\\tt0313792</td>\n",
       "      <td>10</td>\n",
       "      <td>this is a vintage woody allen great film , if ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>ur0550732/</td>\n",
       "      <td>\\tt0490204</td>\n",
       "      <td>10</td>\n",
       "      <td>we start all of our reviews with the following...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16809</th>\n",
       "      <td>ur0386241/</td>\n",
       "      <td>\\tt0118636</td>\n",
       "      <td>6</td>\n",
       "      <td>`` apt pupil '' is an undeniably interesting p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16810</th>\n",
       "      <td>ur0386241/</td>\n",
       "      <td>\\tt0083972</td>\n",
       "      <td>4</td>\n",
       "      <td>this third entry looks cheap , lacks suspense ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16811</th>\n",
       "      <td>ur0386241/</td>\n",
       "      <td>\\tt0076809</td>\n",
       "      <td>3</td>\n",
       "      <td>this film consists mostly of pointless scenes ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16812</th>\n",
       "      <td>ur0386241/</td>\n",
       "      <td>\\tt0064757</td>\n",
       "      <td>3</td>\n",
       "      <td>i find it puzzling that lazenby has so many th...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16813</th>\n",
       "      <td>ur0386241/</td>\n",
       "      <td>\\tt0234354</td>\n",
       "      <td>5</td>\n",
       "      <td>the box-office failure of `` novocaine '' is n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16814</th>\n",
       "      <td>ur0386241/</td>\n",
       "      <td>\\tt0095889</td>\n",
       "      <td>4</td>\n",
       "      <td>the change of setting and the introduction of ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16815</th>\n",
       "      <td>ur0386241/</td>\n",
       "      <td>\\tt0067741</td>\n",
       "      <td>7</td>\n",
       "      <td>casting aside `` shaft '' 's social and histor...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16816</th>\n",
       "      <td>ur0386241/</td>\n",
       "      <td>\\tt0059903</td>\n",
       "      <td>5</td>\n",
       "      <td>`` what 's new , pussycat '' marks the writing...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16817</th>\n",
       "      <td>ur0386241/</td>\n",
       "      <td>\\tt0082700</td>\n",
       "      <td>6</td>\n",
       "      <td>as another person has already said , perhaps i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16818</th>\n",
       "      <td>ur0194347/</td>\n",
       "      <td>\\tt0120601</td>\n",
       "      <td>6</td>\n",
       "      <td>being john malkovich * * 1/2 a tough one to ra...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16819</th>\n",
       "      <td>ur0194347/</td>\n",
       "      <td>\\tt0131980</td>\n",
       "      <td>7</td>\n",
       "      <td>bowfinger * * * very amusing , very silly sati...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16820</th>\n",
       "      <td>ur0194347/</td>\n",
       "      <td>\\tt0137363</td>\n",
       "      <td>5</td>\n",
       "      <td>this thriller starts off slow , then picks up ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16821</th>\n",
       "      <td>ur0194347/</td>\n",
       "      <td>\\tt0203230</td>\n",
       "      <td>8</td>\n",
       "      <td>you can count on me * * * 1/2 very enjoyable a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16822</th>\n",
       "      <td>ur0194347/</td>\n",
       "      <td>\\tt0124198</td>\n",
       "      <td>6</td>\n",
       "      <td>caution ! &lt;sssss&gt; spoilers follow ! &lt;sssss&gt; th...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16823</th>\n",
       "      <td>ur0194347/</td>\n",
       "      <td>\\tt0171580</td>\n",
       "      <td>6</td>\n",
       "      <td>nurse betty * * 1/2 very uneven , off-beat , q...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16824</th>\n",
       "      <td>ur0194347/</td>\n",
       "      <td>\\tt0208874</td>\n",
       "      <td>7</td>\n",
       "      <td>joan allen is terrific as the congresswoman wh...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16825</th>\n",
       "      <td>ur0194347/</td>\n",
       "      <td>\\tt0141109</td>\n",
       "      <td>5</td>\n",
       "      <td>this harmless fantasy was unfairly maligned by...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16826</th>\n",
       "      <td>ur0194347/</td>\n",
       "      <td>\\tt0190138</td>\n",
       "      <td>5</td>\n",
       "      <td>the whole nine yards * * story has a lot of go...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16827</th>\n",
       "      <td>ur0194347/</td>\n",
       "      <td>\\tt0162661</td>\n",
       "      <td>7</td>\n",
       "      <td>great effects and each beheading scene is expl...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16828</th>\n",
       "      <td>ur0194347/</td>\n",
       "      <td>\\tt0181875</td>\n",
       "      <td>7</td>\n",
       "      <td>enjoyable , lightweight film based loosely on ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16829</th>\n",
       "      <td>ur0194347/</td>\n",
       "      <td>\\tt0064757</td>\n",
       "      <td>8</td>\n",
       "      <td>on her majesty 's secret service * * * 1/2 geo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16830</th>\n",
       "      <td>ur0194347/</td>\n",
       "      <td>\\tt0125022</td>\n",
       "      <td>5</td>\n",
       "      <td>heartbreakers * * lazy script and slack direct...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16831</th>\n",
       "      <td>ur0194347/</td>\n",
       "      <td>\\tt0195685</td>\n",
       "      <td>7</td>\n",
       "      <td>erin brockovich * * * julia roberts proves onc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16832</th>\n",
       "      <td>ur0194347/</td>\n",
       "      <td>\\tt0145681</td>\n",
       "      <td>5</td>\n",
       "      <td>the bone collector -lrb- 1999 -rrb- * * high c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16833</th>\n",
       "      <td>ur0194347/</td>\n",
       "      <td>\\tt0120863</td>\n",
       "      <td>7</td>\n",
       "      <td>the thin red line * * * the first movie i 've ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16834</th>\n",
       "      <td>ur0194347/</td>\n",
       "      <td>\\tt0124315</td>\n",
       "      <td>8</td>\n",
       "      <td>when this film opened it got mixed reviews so ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16835</th>\n",
       "      <td>ur0194347/</td>\n",
       "      <td>\\tt0200290</td>\n",
       "      <td>7</td>\n",
       "      <td>small time crooks -lrb- 2000 -rrb- * * * extre...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16836</th>\n",
       "      <td>ur0194347/</td>\n",
       "      <td>\\tt0065143</td>\n",
       "      <td>5</td>\n",
       "      <td>this argento mystery is n't bad but i enjoyed ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16837</th>\n",
       "      <td>ur0194347/</td>\n",
       "      <td>\\tt0145660</td>\n",
       "      <td>6</td>\n",
       "      <td>it 's unavoidable to compare a sequel to its p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16838</th>\n",
       "      <td>ur0194347/</td>\n",
       "      <td>\\tt0051459</td>\n",
       "      <td>8</td>\n",
       "      <td>so elizabeth taylor really can act ! &lt;sssss&gt; i...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>16839 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          user_id  product_id  rating  \\\n",
       "0      ur0116181/  \\tt0185937       1   \n",
       "1      ur0116181/  \\tt0169547      10   \n",
       "2      ur0116181/  \\tt0478304       5   \n",
       "3      ur0116181/  \\tt0195685      10   \n",
       "4      ur0116181/  \\tt0217869      10   \n",
       "5      ur0116181/  \\tt0181689      10   \n",
       "6      ur0116181/  \\tt0062622      10   \n",
       "7      ur0116181/  \\tt0155267       9   \n",
       "8      ur0116181/  \\tt0140352      10   \n",
       "9      ur0116181/  \\tt0887912      10   \n",
       "10     ur0116181/  \\tt0372784      10   \n",
       "11     ur0116181/  \\tt1285016      10   \n",
       "12     ur0116181/  \\tt0167404      10   \n",
       "13     ur0116181/  \\tt1152836       7   \n",
       "14     ur0116181/  \\tt0257044       9   \n",
       "15     ur0116181/  \\tt0416449       8   \n",
       "16     ur0116181/  \\tt1568346       9   \n",
       "17     ur0116181/  \\tt0085809      10   \n",
       "18     ur0550732/  \\tt0478311      10   \n",
       "19     ur0550732/  \\tt0467200       9   \n",
       "20     ur0550732/  \\tt0829482      10   \n",
       "21     ur0550732/  \\tt0328107       8   \n",
       "22     ur0550732/  \\tt0343660       9   \n",
       "23     ur0550732/  \\tt0473308       8   \n",
       "24     ur0550732/  \\tt0402894       8   \n",
       "25     ur0550732/  \\tt0203755       9   \n",
       "26     ur0550732/  \\tt0208874       9   \n",
       "27     ur0550732/  \\tt0223897      10   \n",
       "28     ur0550732/  \\tt0313792      10   \n",
       "29     ur0550732/  \\tt0490204      10   \n",
       "...           ...         ...     ...   \n",
       "16809  ur0386241/  \\tt0118636       6   \n",
       "16810  ur0386241/  \\tt0083972       4   \n",
       "16811  ur0386241/  \\tt0076809       3   \n",
       "16812  ur0386241/  \\tt0064757       3   \n",
       "16813  ur0386241/  \\tt0234354       5   \n",
       "16814  ur0386241/  \\tt0095889       4   \n",
       "16815  ur0386241/  \\tt0067741       7   \n",
       "16816  ur0386241/  \\tt0059903       5   \n",
       "16817  ur0386241/  \\tt0082700       6   \n",
       "16818  ur0194347/  \\tt0120601       6   \n",
       "16819  ur0194347/  \\tt0131980       7   \n",
       "16820  ur0194347/  \\tt0137363       5   \n",
       "16821  ur0194347/  \\tt0203230       8   \n",
       "16822  ur0194347/  \\tt0124198       6   \n",
       "16823  ur0194347/  \\tt0171580       6   \n",
       "16824  ur0194347/  \\tt0208874       7   \n",
       "16825  ur0194347/  \\tt0141109       5   \n",
       "16826  ur0194347/  \\tt0190138       5   \n",
       "16827  ur0194347/  \\tt0162661       7   \n",
       "16828  ur0194347/  \\tt0181875       7   \n",
       "16829  ur0194347/  \\tt0064757       8   \n",
       "16830  ur0194347/  \\tt0125022       5   \n",
       "16831  ur0194347/  \\tt0195685       7   \n",
       "16832  ur0194347/  \\tt0145681       5   \n",
       "16833  ur0194347/  \\tt0120863       7   \n",
       "16834  ur0194347/  \\tt0124315       8   \n",
       "16835  ur0194347/  \\tt0200290       7   \n",
       "16836  ur0194347/  \\tt0065143       5   \n",
       "16837  ur0194347/  \\tt0145660       6   \n",
       "16838  ur0194347/  \\tt0051459       8   \n",
       "\n",
       "                                          review_content  \n",
       "0      sometimes popular opinion really sucks about a...  \n",
       "1      this is an amazing piece of work and it probab...  \n",
       "2      this movie does not belong in a cinema . <ssss...  \n",
       "3      the first nominees for next year 's oscars in ...  \n",
       "4      my expectations for writer/director m. night s...  \n",
       "5      let me warn you that a.i. was my favorite film...  \n",
       "6      this was the movie that made me fall in love w...  \n",
       "7      i do n't want to jinx this movie , but this is...  \n",
       "8      this film is unavoidably being compared to dir...  \n",
       "9      without a doubt this is the best war film sinc...  \n",
       "10     having seen this film a 2nd time only confirms...  \n",
       "11     like others before me when i initially heard a...  \n",
       "12     the word of mouth -lrb- wom -rrb- on this movi...  \n",
       "13     maybe this movie suffers from comparisons with...  \n",
       "14     sam mendes 's directorial follow up to `` amer...  \n",
       "15     this is n't the godfather or close encounters ...  \n",
       "16     the novel & the swedish movie are not great by...  \n",
       "17     i put this film in the same league as `` 2001 ...  \n",
       "18     we start all of our reviews with the following...  \n",
       "19     we start all of our reviews with the following...  \n",
       "20     we start all of our reviews with the following...  \n",
       "21     denzel washington and dakota fanning both dese...  \n",
       "22     these two have it going . <sssss> in this thei...  \n",
       "23     we start all of our reviews with the following...  \n",
       "24     we start all of our reviews with the following...  \n",
       "25     she is one of the sexiest actresses in the mov...  \n",
       "26     what a wonderful political movie . <sssss> it ...  \n",
       "27     we loved everything about this movie , from th...  \n",
       "28     this is a vintage woody allen great film , if ...  \n",
       "29     we start all of our reviews with the following...  \n",
       "...                                                  ...  \n",
       "16809  `` apt pupil '' is an undeniably interesting p...  \n",
       "16810  this third entry looks cheap , lacks suspense ...  \n",
       "16811  this film consists mostly of pointless scenes ...  \n",
       "16812  i find it puzzling that lazenby has so many th...  \n",
       "16813  the box-office failure of `` novocaine '' is n...  \n",
       "16814  the change of setting and the introduction of ...  \n",
       "16815  casting aside `` shaft '' 's social and histor...  \n",
       "16816  `` what 's new , pussycat '' marks the writing...  \n",
       "16817  as another person has already said , perhaps i...  \n",
       "16818  being john malkovich * * 1/2 a tough one to ra...  \n",
       "16819  bowfinger * * * very amusing , very silly sati...  \n",
       "16820  this thriller starts off slow , then picks up ...  \n",
       "16821  you can count on me * * * 1/2 very enjoyable a...  \n",
       "16822  caution ! <sssss> spoilers follow ! <sssss> th...  \n",
       "16823  nurse betty * * 1/2 very uneven , off-beat , q...  \n",
       "16824  joan allen is terrific as the congresswoman wh...  \n",
       "16825  this harmless fantasy was unfairly maligned by...  \n",
       "16826  the whole nine yards * * story has a lot of go...  \n",
       "16827  great effects and each beheading scene is expl...  \n",
       "16828  enjoyable , lightweight film based loosely on ...  \n",
       "16829  on her majesty 's secret service * * * 1/2 geo...  \n",
       "16830  heartbreakers * * lazy script and slack direct...  \n",
       "16831  erin brockovich * * * julia roberts proves onc...  \n",
       "16832  the bone collector -lrb- 1999 -rrb- * * high c...  \n",
       "16833  the thin red line * * * the first movie i 've ...  \n",
       "16834  when this film opened it got mixed reviews so ...  \n",
       "16835  small time crooks -lrb- 2000 -rrb- * * * extre...  \n",
       "16836  this argento mystery is n't bad but i enjoyed ...  \n",
       "16837  it 's unavoidable to compare a sequel to its p...  \n",
       "16838  so elizabeth taylor really can act ! <sssss> i...  \n",
       "\n",
       "[16839 rows x 4 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load in Word2Vec model\n",
    "\n",
    "We want to use word2vec as our embedding layer. First, we load the w2vmodel that we trained(26.89M).\n",
    "\n",
    "See [how](./word2vec.ipynb) the w2vmodel is trained."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "w2vmodel = Word2Vec.load(args.w2vmodel_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.0832856 ,  0.17038353,  0.13446064,  0.20960386, -0.27410474,\n",
       "        0.04419966, -0.02441939, -0.2884749 ,  0.08365629, -0.3235816 ,\n",
       "       -0.14133385, -0.05964658,  0.07084928,  0.10685515, -0.595156  ,\n",
       "        0.1006216 ,  0.08503133, -0.20746365,  0.37630245,  0.35833853,\n",
       "       -0.06807327,  0.3119023 , -0.29489398,  0.20061803, -0.10797408,\n",
       "       -0.14101546, -0.38280725,  0.054291  , -0.44207925, -0.232832  ,\n",
       "       -0.486326  ,  0.05088602, -0.08845872, -0.21135303, -0.24881552,\n",
       "       -0.1047746 , -0.309808  ,  0.02793502,  0.12140413,  0.08667814,\n",
       "       -0.04950768, -0.05872349,  0.00774467,  0.23892054, -0.32583383,\n",
       "       -0.19085103, -0.08125331, -0.01782029, -0.00889523,  0.09103105,\n",
       "       -0.37290302,  0.08022729, -0.24092077,  0.2355133 , -0.16579767,\n",
       "       -0.07698391, -0.01896097, -0.12648211, -0.12753847, -0.02031306,\n",
       "       -0.03370816,  0.11911461,  0.21125719, -0.13139032,  0.0097007 ,\n",
       "       -0.52950764,  0.07178081, -0.06581753,  0.1629323 ,  0.15020199,\n",
       "        0.22294281,  0.02753206,  0.2888069 , -0.04742526, -0.22130242,\n",
       "        0.0441267 ,  0.46879983, -0.55457526, -0.01162305, -0.30460143,\n",
       "        0.36732194,  0.05675109, -0.23911284,  0.12474174, -0.37924278,\n",
       "       -0.2234276 ,  0.4021835 , -0.06928826, -0.29548535, -0.41033608,\n",
       "       -0.20425427, -0.11497852,  0.33286735, -0.18669274,  0.04382398,\n",
       "       -0.8555307 ,  0.15812753, -0.3903703 , -0.17084396,  0.24208918],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w2vmodel.wv.vectors[1487]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Vocabulary & Vectorizer\n",
    "\n",
    "We write a class `Vectorizer` to help us tokenize text and get word vectors/id from w2vmodel and convert w2vmodel to `nn.Embedding` model straight away.\n",
    "\n",
    "Notice that we have process some common English stop words. `<sssss>`, `--rrb--` and some other tokens are included, which are not much meaningful but commonly seen in the dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "stop_words = ['and', ',', '.', 'ourselves', 'hers', 'between', 'yourself', 'but', 'again', 'there', 'about', 'once', \n",
    "              'during', 'out', 'very', 'having', 'with', 'they', 'own', 'an', 'be', 'some', 'for', 'do', 'its', 'yours', \n",
    "              'such', 'into', 'of', 'most', 'itself', 'other', 'off', 'is', 's', 'am', 'or', 'who', 'as', 'from', 'him',\n",
    "              'each', 'the', 'themselves', 'until', 'below', 'are', 'we', 'these', 'your', 'his', 'me', 'were', 'her',\n",
    "              'himself', 'this', 'should', 'our', 'their', 'above', 'both','to', 'ours', 'had', 'she',  'when', 'at', \n",
    "              'them','been', 'have', 'in', 'will', 'on', 'does', 'yourselves', 'then', 'that', 'because', 'what', 'so', \n",
    "              'can', 'did', 'now', 'he', 'you', 'herself', 'has', 'myself', 'which', 'those', 'i', 'after', 'whom',\n",
    "              'theirs', 'my', 'a', 'by', 'doing', 'it', 'was', '<sssss>', '-rrb-', '-lrb-']\n",
    "\n",
    "class Vectorizer(object):\n",
    "    def __init__(self, model):\n",
    "        self.word_list = model.wv.index2word\n",
    "        self.vector_list = w2vmodel.wv.vectors\n",
    "    \n",
    "    def getVector(self, word):\n",
    "        try:\n",
    "            i = self.word_list.index(word)\n",
    "            return self.vector_list[i]\n",
    "        except:\n",
    "            i = self.word_list.index('unknown')\n",
    "            return self.vector_list[i]\n",
    "        \n",
    "    def getVectorsFromText(self, text):\n",
    "        vectors = []\n",
    "        for word in text.split(\" \"):\n",
    "            if word in stop_words:\n",
    "                continue\n",
    "            \n",
    "            vectors.append(self.getVector(word))\n",
    "        \n",
    "        return vectors\n",
    "    \n",
    "    # Get id of a word\n",
    "    def getId(self, word):\n",
    "        try:\n",
    "            i = self.word_list.index(word)\n",
    "            return i\n",
    "        except:\n",
    "            i = self.word_list.index('unknown')\n",
    "            return i\n",
    "    \n",
    "    # Tokenize the sentences and replace every token with its id\n",
    "    def getIdsFromText(self, text):\n",
    "        ids = []\n",
    "        for word in text.split(\" \"):\n",
    "            if word in stop_words:\n",
    "                continue\n",
    "            \n",
    "            ids.append(self.getId(word))\n",
    "        \n",
    "        return ids\n",
    "    \n",
    "    def getVectorById(self, id):\n",
    "        return self.vector_list[id];\n",
    "    \n",
    "    # Generate nn.Embedding from trained w2vmodel\n",
    "    def getEmbedding(self):\n",
    "        weights = torch.FloatTensor(self.vector_list)\n",
    "        return torch.nn.Embedding.from_pretrained(weights)\n",
    "            \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "voca = Vectorizer(w2vmodel)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[3848, 436, 8560]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "voca.getIdsFromText(\"hello my name is derek\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Customize Dataset\n",
    "\n",
    "To extend `torch.utils.data.Dataset`, we can build a Dataset class which with `DataLoader` will help us load data in batches.\n",
    "\n",
    "In `__getitem__`, we use our `Vectorizer` to process the text into list of id."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MovieReviewDataset(torch.utils.data.Dataset):\n",
    "    def __init__(self, df, vectorizer: Vectorizer, text_col = 'review_content', rating_col = 'rating'):\n",
    "        self.df = df\n",
    "        self.text_col = text_col\n",
    "        self.rating_col = rating_col\n",
    "        self.vectorizer = vectorizer\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.df.index)\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        line = self.df.iloc[idx]\n",
    "        text = torch.tensor(self.vectorizer.getIdsFromText(line[self.text_col]))\n",
    "        rating = torch.tensor(line[self.rating_col])\n",
    "        \n",
    "        return {'text': text, 'rating': rating}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create both training and validation datasets\n",
    "\n",
    "dataframes = {'training': training_df, 'validation': validation_df}\n",
    "\n",
    "datasets = {x: MovieReviewDataset(dataframes[x], voca)\n",
    "              for x in ['training', 'validation']}\n",
    "\n",
    "dataset_sizes = {x: len(datasets[x]) for x in ['training', 'validation']}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(1487)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "datasets['training'].__getitem__(4)['text'][1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build DataLoader\n",
    "\n",
    "As mentioned, `DataLoader` helps us to load data in batches while training, which will make our training process more gpu-efficient thus faster. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Costumize `DataLoader` batch format\n",
    "def variable_size_collate(batch):\n",
    "    ratings = []\n",
    "    longest_len = 0\n",
    "    texts = []\n",
    "    for item in batch:\n",
    "        thislen = len(item['text'])\n",
    "        longest_len = thislen if thislen > longest_len else longest_len\n",
    "        ratings.append(item['rating'])\n",
    "    \n",
    "    for i in range(longest_len):\n",
    "        pos =  []\n",
    "        for item in batch:\n",
    "            text = item['text']\n",
    "            if i < len(text):\n",
    "                pos.append(text[i])\n",
    "            else:\n",
    "                pos.append(torch.tensor(0))\n",
    "        pos = torch.stack(pos)\n",
    "        texts.append(pos)\n",
    "        \n",
    "    ratings = torch.FloatTensor(ratings)\n",
    "    texts = torch.stack(texts)\n",
    "    return {'text': texts, 'rating': ratings}\n",
    "\n",
    "dataloaders = {x: torch.utils.data.DataLoader(datasets[x], batch_size=args.batch_size, shuffle=True, num_workers=args.num_workers, collate_fn=variable_size_collate)\n",
    "              for x in ['training', 'validation']}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get Embedding Layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Embedding(21750, 100)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embedding = voca.getEmbedding()\n",
    "embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.0325,  0.1499,  0.2310,  0.0769,  0.0743,  0.1067,  0.2858, -0.5892,\n",
       "          0.4588, -0.4283,  0.2836, -0.2005,  0.2104,  0.1039, -0.3846,  0.0301,\n",
       "         -0.1920, -0.1838,  0.1904,  0.2130,  0.2405, -0.0829,  0.1903,  0.4234,\n",
       "         -0.1527, -0.2340, -0.2299,  0.0274,  0.0333,  0.2278, -0.0440, -0.1399,\n",
       "         -0.1714, -0.4231, -0.4074, -0.3462,  0.2565, -0.0600,  0.0391, -0.2285,\n",
       "         -0.2423,  0.0792,  0.4543,  0.4119,  0.1438,  0.2681,  0.0897,  0.5281,\n",
       "         -0.2478,  0.1927, -0.3498,  0.0594,  0.0254,  0.2006, -0.3440, -0.0786,\n",
       "          0.0687,  0.3086, -0.0354,  0.0525,  0.0537, -0.1680, -0.0263, -0.0323,\n",
       "         -0.1826, -0.1126, -0.0501, -0.1170,  0.0148, -0.4299,  0.2880,  0.0525,\n",
       "         -0.1989, -0.0569,  0.1596,  0.0896,  0.2335, -0.1160, -0.5452, -0.3574,\n",
       "          0.2767,  0.0259, -0.1858, -0.0766,  0.1818, -0.2650, -0.0564,  0.1181,\n",
       "         -0.0819, -0.1206, -0.2344, -0.2920,  0.4595, -0.2293,  0.0395, -0.4779,\n",
       "         -0.0076, -0.4342, -0.0361,  0.1824],\n",
       "        [ 0.0728,  0.4475,  0.2889,  0.6581, -0.2154, -0.1692,  0.1175, -0.4225,\n",
       "          0.2876, -0.0069, -0.0110, -0.2158,  0.1124,  0.3095, -0.4909, -0.0374,\n",
       "          0.2908, -0.2362,  0.3220,  0.3031,  0.1976,  0.4543, -0.2763,  0.4242,\n",
       "         -0.0675,  0.3043,  0.0496, -0.1591, -0.0030,  0.0591, -0.2991, -0.1259,\n",
       "         -0.0108, -0.4180, -0.1946, -0.2087, -0.2640, -0.3012,  0.0754, -0.1063,\n",
       "         -0.2697, -0.0570,  0.1064,  0.3753, -0.2486, -0.1595,  0.0162,  0.4158,\n",
       "         -0.1029,  0.0843, -0.0268, -0.2629, -0.0029,  0.1295, -0.3243, -0.4039,\n",
       "         -0.3021,  0.0189,  0.0405,  0.1207,  0.0067,  0.0385, -0.0292,  0.0688,\n",
       "          0.1279, -0.1024,  0.0298, -0.1025, -0.0892, -0.4037,  0.0144,  0.4460,\n",
       "          0.2058,  0.0189, -0.0835,  0.4565,  0.6659, -0.1841, -0.5783, -0.4331,\n",
       "          0.2310, -0.0523, -0.4869,  0.3042,  0.0993,  0.1298,  0.2515,  0.3412,\n",
       "         -0.0292, -0.2744, -0.3194, -0.3539,  0.6329, -0.2984,  0.1126, -0.3027,\n",
       "         -0.1602, -0.5661, -0.1661,  0.5569],\n",
       "        [ 0.1075,  0.2973,  0.0429,  0.0982, -0.0120, -0.0180,  0.1480, -0.3428,\n",
       "          0.4673, -0.3196,  0.1240, -0.0519,  0.1013,  0.0716, -0.1890, -0.0845,\n",
       "         -0.0484, -0.1590, -0.0191,  0.1408,  0.0613,  0.0296,  0.3083,  0.2020,\n",
       "         -0.0317, -0.0572, -0.2578,  0.1258,  0.0955, -0.0585,  0.0235, -0.1146,\n",
       "          0.0252, -0.1850, -0.2485, -0.3119,  0.1190,  0.0449, -0.1762, -0.1599,\n",
       "         -0.0408, -0.2324,  0.3396,  0.1599, -0.0783,  0.2365,  0.3121,  0.2694,\n",
       "         -0.1603,  0.2445, -0.1240, -0.0147, -0.1657,  0.2639, -0.2581,  0.1353,\n",
       "          0.0786,  0.2130,  0.0867, -0.2103, -0.0305,  0.0446,  0.0673, -0.2930,\n",
       "          0.0412, -0.0589,  0.0915, -0.1463, -0.1058, -0.1730,  0.0368,  0.3176,\n",
       "         -0.0469,  0.1403,  0.0579,  0.3128,  0.2174, -0.0979, -0.3067, -0.2910,\n",
       "          0.2438, -0.1418, -0.2653, -0.0149, -0.1827, -0.0604,  0.2390,  0.4305,\n",
       "          0.0401, -0.1878, -0.3765, -0.0764,  0.4162, -0.2146,  0.1464, -0.5520,\n",
       "         -0.1611, -0.2404, -0.2840,  0.2630],\n",
       "        [-0.2829, -0.1425,  0.2868,  0.4539, -0.2175,  0.0114,  0.2716, -0.7900,\n",
       "          0.0619, -0.3274,  0.1106, -0.1428, -0.0979,  0.3277, -0.8046, -0.0856,\n",
       "          0.0055, -0.0606,  0.1229,  0.5086,  0.2590, -0.1125,  0.2055,  0.4095,\n",
       "         -0.0194, -0.1394, -0.2934, -0.1185,  0.0455, -0.2375, -0.3078,  0.0446,\n",
       "         -0.1019, -0.1904, -0.4611, -0.4710,  0.0831, -0.5158, -0.1553, -0.2029,\n",
       "         -0.0252, -0.1295,  0.4573,  0.0833, -0.2978, -0.1550,  0.3897,  0.4923,\n",
       "         -0.1506,  0.1597, -0.3415,  0.2899, -0.2123,  0.1637, -0.2821, -0.0806,\n",
       "          0.3367,  0.2711,  0.2232, -0.0940, -0.0464, -0.1270, -0.1046, -0.1924,\n",
       "         -0.2585, -0.1654, -0.2082,  0.0733,  0.2387, -0.2035,  0.4740,  0.0903,\n",
       "         -0.1721, -0.2503,  0.3687,  0.0408,  0.4846, -0.2941, -0.1780, -0.4259,\n",
       "          0.2984, -0.1923, -0.1762, -0.0923,  0.2511, -0.6914,  0.2776, -0.0905,\n",
       "         -0.3042, -0.1704, -0.1186, -0.1511,  0.1403, -0.2633,  0.0637, -0.4635,\n",
       "          0.2604, -0.4139, -0.3479,  0.0909],\n",
       "        [-0.0402,  0.5730,  0.4137, -0.1693, -0.1945,  0.2099, -0.0794, -0.3981,\n",
       "          0.0703, -0.4492, -0.0091, -0.1249, -0.0580,  0.3433, -0.6541,  0.0549,\n",
       "          0.4216, -0.0104,  0.2225,  0.2777, -0.1346,  0.5041,  0.0208,  0.2356,\n",
       "         -0.1498,  0.0423,  0.0807,  0.0321, -0.4733, -0.1404, -0.3194, -0.0536,\n",
       "          0.1457, -0.4023, -0.0927, -0.0553,  0.3094, -0.3521,  0.0787, -0.0868,\n",
       "         -0.0601, -0.1926,  0.1415,  0.0662, -0.0372, -0.5976,  0.3841,  0.2668,\n",
       "          0.0562,  0.0287, -0.2799, -0.2729, -0.2298,  0.1043, -0.1540,  0.1048,\n",
       "         -0.1823,  0.0521, -0.6007, -0.0989, -0.0975,  0.0156,  0.1842,  0.1388,\n",
       "         -0.0360, -0.2094, -0.1312, -0.0316, -0.0761,  0.3240,  0.2941,  0.1452,\n",
       "          0.5350, -0.2045,  0.0392,  0.2343,  0.5287, -0.2451, -0.3568, -0.1351,\n",
       "          0.4041,  0.1713,  0.1433,  0.1013,  0.3352, -0.4758,  0.1617,  0.0023,\n",
       "         -0.0418, -0.3242, -0.5170, -0.1224,  0.3048, -0.2492, -0.3647, -0.4544,\n",
       "          0.2019, -0.4558, -0.3122,  0.0800],\n",
       "        [-0.4895,  0.3313,  0.6343, -0.0755, -0.2271, -0.0889, -0.2018, -0.6555,\n",
       "          0.3411, -0.5774,  0.2079, -0.3073,  0.1436,  0.2710, -0.6811,  0.2006,\n",
       "         -0.0328, -0.3078,  0.2432,  0.4249,  0.3218,  0.0268,  0.2460,  0.2934,\n",
       "          0.0728, -0.3656, -0.2104, -0.0436,  0.0365,  0.0551, -0.1941, -0.1557,\n",
       "         -0.0638, -0.4460,  0.0499, -0.3140,  0.0247, -0.1974,  0.0199, -0.1258,\n",
       "         -0.0429,  0.0597,  0.5257,  0.3165,  0.2447,  0.0245,  0.1113,  0.6310,\n",
       "         -0.1804,  0.2301, -0.3177, -0.1609,  0.0394,  0.5476, -0.0331, -0.1142,\n",
       "          0.1754,  0.0794, -0.1180, -0.0432,  0.1411,  0.3022, -0.0596, -0.0486,\n",
       "          0.0012, -0.2823, -0.2501,  0.1249,  0.1406, -0.4929,  0.3161, -0.0744,\n",
       "         -0.0680, -0.2633,  0.1342, -0.1622,  0.2110, -0.2160, -0.2770, -0.4804,\n",
       "          0.3882,  0.1349, -0.0355, -0.1504,  0.2254, -0.4205,  0.1459,  0.1846,\n",
       "         -0.0696, -0.1876,  0.0334, -0.0570,  0.6631, -0.4299, -0.1738, -0.3416,\n",
       "         -0.0970, -0.3424, -0.1496,  0.3769],\n",
       "        [ 0.0833,  0.1704,  0.1345,  0.2096, -0.2741,  0.0442, -0.0244, -0.2885,\n",
       "          0.0837, -0.3236, -0.1413, -0.0596,  0.0708,  0.1069, -0.5952,  0.1006,\n",
       "          0.0850, -0.2075,  0.3763,  0.3583, -0.0681,  0.3119, -0.2949,  0.2006,\n",
       "         -0.1080, -0.1410, -0.3828,  0.0543, -0.4421, -0.2328, -0.4863,  0.0509,\n",
       "         -0.0885, -0.2114, -0.2488, -0.1048, -0.3098,  0.0279,  0.1214,  0.0867,\n",
       "         -0.0495, -0.0587,  0.0077,  0.2389, -0.3258, -0.1909, -0.0813, -0.0178,\n",
       "         -0.0089,  0.0910, -0.3729,  0.0802, -0.2409,  0.2355, -0.1658, -0.0770,\n",
       "         -0.0190, -0.1265, -0.1275, -0.0203, -0.0337,  0.1191,  0.2113, -0.1314,\n",
       "          0.0097, -0.5295,  0.0718, -0.0658,  0.1629,  0.1502,  0.2229,  0.0275,\n",
       "          0.2888, -0.0474, -0.2213,  0.0441,  0.4688, -0.5546, -0.0116, -0.3046,\n",
       "          0.3673,  0.0568, -0.2391,  0.1247, -0.3792, -0.2234,  0.4022, -0.0693,\n",
       "         -0.2955, -0.4103, -0.2043, -0.1150,  0.3329, -0.1867,  0.0438, -0.8555,\n",
       "          0.1581, -0.3904, -0.1708,  0.2421]])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embedding(torch.LongTensor(voca.getIdsFromText(\"hello my name is derek trust issue hey what the fuck is that\")))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define LSTM Structure\n",
    "\n",
    "We get to design our own LSTM model by writing class extending `nn.Module`. I have this structure in mind: Embedding layer + LSTM(bidirectional) + Linear. \n",
    "\n",
    "To see the introduction of LSTM, check out the other [notebook](./rating_model_fastai.ipynb#LSTM)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LSTM(nn.Module):\n",
    "    def __init__(self, input_dim, embedding_dim, hidden_dim, output_dim, bidirectional = False):\n",
    "        \n",
    "        super().__init__()\n",
    "        \n",
    "        self.embedding = nn.Embedding(input_dim, embedding_dim)\n",
    "        \n",
    "        self.rnn = nn.LSTM(embedding_dim, hidden_dim, bidirectional=bidirectional)\n",
    "        \n",
    "        self.fc = nn.Linear(hidden_dim *  (2 if bidirectional else 1) , output_dim)\n",
    "        \n",
    "    def forward(self, text):\n",
    "\n",
    "        #text = [sent len, batch size]\n",
    "        \n",
    "        embedded = self.embedding(text)\n",
    "        \n",
    "        #embedded = [sent len, batch size, emb dim]\n",
    "        \n",
    "        output, hidden = self.rnn(embedded)\n",
    "        \n",
    "        #output = [sent len, batch size, hid dim]\n",
    "        #hidden = [1, batch size, hid dim]\n",
    "        \n",
    "        return self.fc(output[-1,:,:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "INPUT_DIM = embedding.num_embeddings\n",
    "EMBEDDING_DIM = embedding.embedding_dim\n",
    "HIDDEN_DIM = 256\n",
    "OUTPUT_DIM = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = LSTM(INPUT_DIM, EMBEDDING_DIM, HIDDEN_DIM, OUTPUT_DIM, bidirectional=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.to(args.device)\n",
    "# Replace embedding layer with our trained w2v embedding layer\n",
    "model.embedding = embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The model has 733,697 trainable parameters\n"
     ]
    }
   ],
   "source": [
    "def count_parameters(model):\n",
    "    return sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
    "\n",
    "print(f'The model has {count_parameters(model):,} trainable parameters')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Init Optimizer & Criterion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.optim as optim\n",
    "\n",
    "optimizer = optim.SGD(model.parameters(), lr=1e-3)\n",
    "lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=7, gamma=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Here we use MSELoss, in the training process down below, we altered this into RMSE as required\n",
    "criterion = nn.MSELoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = model.to(args.device)\n",
    "criterion = criterion.to(args.device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model(model: nn.Module, criterion, optimizer, scheduler, num_epochs=5):\n",
    "    since = time.time()\n",
    "    \n",
    "    best_model_wts = copy.deepcopy(model.state_dict())\n",
    "    best_loss = float(\"inf\")\n",
    "    \n",
    "    for epoch in range(num_epochs):\n",
    "        print('Epoch {}/{}'.format(epoch, num_epochs - 1))\n",
    "        print('-' * 10)\n",
    "        \n",
    "        for phase in ['training', 'validation']:\n",
    "            if phase == 'training':\n",
    "                scheduler.step()\n",
    "                model.train()\n",
    "            else:\n",
    "                model.eval()\n",
    "            \n",
    "            running_loss = 0.0\n",
    "            # running_corrects = 0\n",
    "            \n",
    "            for data in dataloaders[phase]:\n",
    "                texts = data['text'].to(args.device)\n",
    "                ratings = data['rating'].to(args.device)\n",
    "                \n",
    "                optimizer.zero_grad()\n",
    "                \n",
    "                with torch.set_grad_enabled(phase == 'training'):\n",
    "                    preds = model(texts)\n",
    "                    \n",
    "                    # Alter the MSE to RMSE by adding the sqrt computation\n",
    "                    loss = torch.sqrt(criterion(preds, ratings))\n",
    "                    \n",
    "                    if phase == 'training':\n",
    "                        loss.backward()\n",
    "                        optimizer.step()\n",
    "                \n",
    "                running_loss += loss.item() * texts.size(1)\n",
    "                # running_corrects += torch.sum(torch.round(preds) == ratings)\n",
    "                \n",
    "            epoch_loss = running_loss / dataset_sizes[phase]\n",
    "            # epoch_acc = running_corrects.double() / dataset_sizes[phase]\n",
    "            \n",
    "            print('{} Loss: {:.4f}'.format(\n",
    "                phase, epoch_loss))\n",
    "\n",
    "            # deep copy the model\n",
    "            if phase == 'validation' and epoch_loss < best_loss:\n",
    "                best_loss = epoch_loss\n",
    "                best_model_wts = copy.deepcopy(model.state_dict())\n",
    "                \n",
    "        print()\n",
    "    \n",
    "    time_elapsed = time.time() - since\n",
    "    print('Training complete in {:.0f}m {:.0f}s'.format(\n",
    "            time_elapsed // 60, time_elapsed % 60))\n",
    "    print('Best val Loss: {:4f}'.format(best_loss))\n",
    "\n",
    "    # load best model weights\n",
    "    model.load_state_dict(best_model_wts)\n",
    "    return model  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0/9\n",
      "----------\n",
      "training Loss: 3.6331\n",
      "validation Loss: 2.2946\n",
      "\n",
      "Epoch 1/9\n",
      "----------\n",
      "training Loss: 2.3042\n",
      "validation Loss: 2.2925\n",
      "\n",
      "Epoch 2/9\n",
      "----------\n",
      "training Loss: 2.2967\n",
      "validation Loss: 2.2900\n",
      "\n",
      "Epoch 3/9\n",
      "----------\n",
      "training Loss: 2.2918\n",
      "validation Loss: 2.2824\n",
      "\n",
      "Epoch 4/9\n",
      "----------\n",
      "training Loss: 2.2914\n",
      "validation Loss: 2.2759\n",
      "\n",
      "Epoch 5/9\n",
      "----------\n",
      "training Loss: 2.2872\n",
      "validation Loss: 2.2746\n",
      "\n",
      "Epoch 6/9\n",
      "----------\n",
      "training Loss: 2.2881\n",
      "validation Loss: 2.2749\n",
      "\n",
      "Epoch 7/9\n",
      "----------\n",
      "training Loss: 2.2886\n",
      "validation Loss: 2.2784\n",
      "\n",
      "Epoch 8/9\n",
      "----------\n",
      "training Loss: 2.2874\n",
      "validation Loss: 2.2795\n",
      "\n",
      "Epoch 9/9\n",
      "----------\n",
      "training Loss: 2.2848\n",
      "validation Loss: 2.2755\n",
      "\n",
      "Training complete in 24m 52s\n",
      "Best val Loss: 2.274637\n"
     ]
    }
   ],
   "source": [
    "model = train_model(model, criterion, optimizer, lr_scheduler,\n",
    "                       num_epochs=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Predict Helper Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(text):\n",
    "    intens = torch.tensor([[i] for i in voca.getIdsFromText(text)]).to(args.device)\n",
    "    outtens = model(intens)\n",
    "    rating = round(outtens[0].item());\n",
    "    if rating < 1:\n",
    "        rating = 1\n",
    "    elif rating > 10:\n",
    "        rating = 10\n",
    "    return rating\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predict(\"haha good bad so\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/admin/.local/lib/python3.6/site-packages/torch/serialization.py:250: UserWarning: Couldn't retrieve source code for container of type LSTM. It won't be checked for correctness upon loading.\n",
      "  \"type \" + obj.__name__ + \". It won't be checked \"\n"
     ]
    }
   ],
   "source": [
    "torch.save(model, './trained/stage-1.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "model2 = torch.load('./trained/stage-1.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict2(text):\n",
    "    intens = torch.tensor([[i] for i in voca.getIdsFromText(text)]).to(args.device)\n",
    "    outtens = model2(intens)\n",
    "    rating = round(outtens[0].item());\n",
    "    if rating < 1:\n",
    "        rating = 1\n",
    "    elif rating > 10:\n",
    "        rating = 10\n",
    "    return rating\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predict2('not worth it')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Predict on Test Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "ratings = []\n",
    "for line in test_df['review_content']:\n",
    "    ratings.append(predict2(line))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_data = {'review_content': test_df['review_content'], 'rating': ratings}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_df = pd.DataFrame(output_data)\n",
    "output_df.to_csv('senti_output.ss', sep='\\t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Conclusion\n",
    "\n",
    "**Sizes:**\n",
    "\n",
    "Language Model Size(word2vec):       26.89M\n",
    "\n",
    "Classifier Model Size:               11.38M\n",
    "\n",
    "Total Size:                          38.27M\n",
    "\n",
    "**Loss:**\n",
    "\n",
    "RMSE:                              2.274637\n",
    "\n",
    "**Test Set Output:**\n",
    "\n",
    "File:            senti_output.ss (discarded)\n",
    "\n",
    "**Comment:**\n",
    "\n",
    "This model is more like a \"built from scratch\" attempt. The Word2Vec model was trained from a very limited corpus and the LSTM model structure itself is rather simple and not very much thought out. So this model expectedly performs much worse than the `AWD_LSTM` model that I trained using fastai in another [notebook](./rating_model_fastai.ipynb) as \"Plan B\".\n",
    "\n",
    "- the language model I trained is pretty tiny with the size of only 27M, compared to 191.32M of \"Plan B\". As a result, the language model as the first layer of the network performs poorly right in front of the head, making it barely possible to train a good classifier.\n",
    "- the LSTM model I designed is way too simple with the size of only 11.38M, compared to 487.05M of \"Plan B\".\n",
    "\n",
    "But even with a small size of language model and LSTM model like that, it still take a machine equiped with an M40 graphic card a relatively long time to train. I'm guessing the main reason is that the model is trained starting from 'zero_state'. In terms of \"Plan B\", transfer learning comes with an advantage of preloaded already-trained weights, which makes the training process shorter."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}