123028_Assignment7.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 5504,
     "status": "ok",
     "timestamp": 1682613921693,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "1iTa7M9GRTfP",
    "outputId": "4a9d3b22-7e54-454b-a8e1-13a55f41e38d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda\n"
     ]
    }
   ],
   "source": [
    "import torch, torchdata, torchtext\n",
    "from torch import nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "import random, math, time\n",
    "\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "print(device)\n",
    "\n",
    "#make our work comparable if restarted the kernel\n",
    "SEED = 777\n",
    "torch.manual_seed(SEED)\n",
    "torch.backends.cudnn.deterministic = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['http_proxy']  = 'http://192.41.170.23:3128'\n",
    "os.environ['https_proxy'] = 'http://192.41.170.23:3128'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 330
    },
    "executionInfo": {
     "elapsed": 90,
     "status": "error",
     "timestamp": 1682613921693,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "_SeneBiNTArh",
    "outputId": "f117db45-119b-4016-9f62-7f540511e145"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'NVIDIA GeForce RTX 2080 Ti'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.cuda.get_device_name(0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "R_FLp3W5SgHP"
   },
   "source": [
    "\n",
    "# 1. ETL: Load Dataset\n",
    "ref. of the dataset https://www.kaggle.com/datasets/ratthachat/scb-mt-enth-2020?resource=download"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "executionInfo": {
     "elapsed": 90,
     "status": "aborted",
     "timestamp": 1682613921695,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "wIKaSvTGSE-O"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>en</th>\n",
       "      <th>th</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>These are good for using at night.</td>\n",
       "      <td>ของพวกนี้ดีต่อการใช้งานตอนกลางคืน</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>It's Max Smith.</td>\n",
       "      <td>แม็กซ์ สมิธค่ะ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>please bring your labcoat today.give me a miss...</td>\n",
       "      <td>ช่วยเอาเสื้อโค๊ตห้องแลปของเธอมาทีนะวันนี้ มิสค...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Scotch-Brite XN002033480 Wet Mop</td>\n",
       "      <td>ม็อบเปียกซับน้ําดีเยี่ยม สก๊อตช์-ไบรต์ XN00203...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Coupon Detail 500 units for coupons for any jo...</td>\n",
       "      <td>500 หน่วยสําหรับคูปองสําหรับการสั่งซื้อ joybuy...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100172</th>\n",
       "      <td>I didn't really care for the item.</td>\n",
       "      <td>ฉันไม่ได้สนใจรายการ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100173</th>\n",
       "      <td>Worth a watch!</td>\n",
       "      <td>คุ้มค่ากับการดู!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100174</th>\n",
       "      <td>Section::::Paralympics. Sitting volleyball was...</td>\n",
       "      <td>วอลเลย์บอลนั่ง () บางครั้งเรียกในชื่อ วอลเลย์บ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100175</th>\n",
       "      <td>Haha no just saying! Btw don't open the front ...</td>\n",
       "      <td>ฮะฮ่าไม่แค่พูดอ่ะ ยังไงก็ตามไม่เปิดข้างหน้าอาจ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100176</th>\n",
       "      <td>Well done, Samsung!</td>\n",
       "      <td>ทําได้ดีมาก Samsung!</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100177 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                       en   \n",
       "0                      These are good for using at night.  \\\n",
       "1                                         It's Max Smith.   \n",
       "2       please bring your labcoat today.give me a miss...   \n",
       "3                        Scotch-Brite XN002033480 Wet Mop   \n",
       "4       Coupon Detail 500 units for coupons for any jo...   \n",
       "...                                                   ...   \n",
       "100172                 I didn't really care for the item.   \n",
       "100173                                     Worth a watch!   \n",
       "100174  Section::::Paralympics. Sitting volleyball was...   \n",
       "100175  Haha no just saying! Btw don't open the front ...   \n",
       "100176                                Well done, Samsung!   \n",
       "\n",
       "                                                       th  \n",
       "0                       ของพวกนี้ดีต่อการใช้งานตอนกลางคืน  \n",
       "1                                          แม็กซ์ สมิธค่ะ  \n",
       "2       ช่วยเอาเสื้อโค๊ตห้องแลปของเธอมาทีนะวันนี้ มิสค...  \n",
       "3       ม็อบเปียกซับน้ําดีเยี่ยม สก๊อตช์-ไบรต์ XN00203...  \n",
       "4       500 หน่วยสําหรับคูปองสําหรับการสั่งซื้อ joybuy...  \n",
       "...                                                   ...  \n",
       "100172                                ฉันไม่ได้สนใจรายการ  \n",
       "100173                                   คุ้มค่ากับการดู!  \n",
       "100174  วอลเลย์บอลนั่ง () บางครั้งเรียกในชื่อ วอลเลย์บ...  \n",
       "100175  ฮะฮ่าไม่แค่พูดอ่ะ ยังไงก็ตามไม่เปิดข้างหน้าอาจ...  \n",
       "100176                               ทําได้ดีมาก Samsung!  \n",
       "\n",
       "[100177 rows x 2 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "data_raw = pd.read_csv(\"scb_mt_enth_2020_test.csv\")[['en', 'th']]\n",
    "data_raw"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "executionInfo": {
     "elapsed": 91,
     "status": "aborted",
     "timestamp": 1682613921696,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "xzSlyHTlUCFu"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>en</th>\n",
       "      <th>th</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>100172</th>\n",
       "      <td>I didn't really care for the item.</td>\n",
       "      <td>ฉันไม่ได้สนใจรายการ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100173</th>\n",
       "      <td>Worth a watch!</td>\n",
       "      <td>คุ้มค่ากับการดู!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100174</th>\n",
       "      <td>Section::::Paralympics. Sitting volleyball was...</td>\n",
       "      <td>วอลเลย์บอลนั่ง () บางครั้งเรียกในชื่อ วอลเลย์บ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100175</th>\n",
       "      <td>Haha no just saying! Btw don't open the front ...</td>\n",
       "      <td>ฮะฮ่าไม่แค่พูดอ่ะ ยังไงก็ตามไม่เปิดข้างหน้าอาจ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100176</th>\n",
       "      <td>Well done, Samsung!</td>\n",
       "      <td>ทําได้ดีมาก Samsung!</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                       en   \n",
       "100172                 I didn't really care for the item.  \\\n",
       "100173                                     Worth a watch!   \n",
       "100174  Section::::Paralympics. Sitting volleyball was...   \n",
       "100175  Haha no just saying! Btw don't open the front ...   \n",
       "100176                                Well done, Samsung!   \n",
       "\n",
       "                                                       th  \n",
       "100172                                ฉันไม่ได้สนใจรายการ  \n",
       "100173                                   คุ้มค่ากับการดู!  \n",
       "100174  วอลเลย์บอลนั่ง () บางครั้งเรียกในชื่อ วอลเลย์บ...  \n",
       "100175  ฮะฮ่าไม่แค่พูดอ่ะ ยังไงก็ตามไม่เปิดข้างหน้าอาจ...  \n",
       "100176                               ทําได้ดีมาก Samsung!  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_raw.dropna(inplace=True)\n",
    "data_raw.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "oftONXyuVBjX"
   },
   "source": [
    "# 2. EDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "executionInfo": {
     "elapsed": 96,
     "status": "aborted",
     "timestamp": 1682613921701,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "zT7UqS4OU95l"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "100177"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset = data_raw.values.tolist()\n",
    "dataset_size = len(dataset)\n",
    "dataset_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "executionInfo": {
     "elapsed": 98,
     "status": "aborted",
     "timestamp": 1682613921703,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "yEfoNRWDVEn6"
   },
   "outputs": [],
   "source": [
    "random.shuffle(dataset)\n",
    "train_data_set_size = int(dataset_size *0.9)\n",
    "\n",
    "train_dataset = dataset[:train_data_set_size]\n",
    "test = dataset[train_data_set_size:]\n",
    "\n",
    "train_size = int(train_data_set_size *0.9)\n",
    "train = train_dataset[:train_size]\n",
    "val = train_dataset[train_size:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "executionInfo": {
     "elapsed": 102,
     "status": "aborted",
     "timestamp": 1682613921707,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "Z7bwFd20VG57"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(81143, 9016, 10018)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train),len(val),len(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "cX7Q9bT2VJrZ"
   },
   "source": [
    "# 3. Preprocessing "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "executionInfo": {
     "elapsed": 103,
     "status": "aborted",
     "timestamp": 1682613921708,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "6LD3mwOJVI1l"
   },
   "outputs": [],
   "source": [
    "SRC_LANGUAGE = 'th'\n",
    "TRG_LANGUAGE = 'en'\n",
    "# Place-holders\n",
    "token_transform = {}\n",
    "vocab_transform = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "executionInfo": {
     "elapsed": 104,
     "status": "aborted",
     "timestamp": 1682613921709,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "5k7NNFQ1VV1G"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting attacut\n",
      "  Downloading attacut-1.0.6-py3-none-any.whl (1.3 MB)\n",
      "     |████████████████████████████████| 1.3 MB 1.1 MB/s            \n",
      "\u001b[?25hRequirement already satisfied: numpy>=1.17.0 in /opt/conda/lib/python3.9/site-packages (from attacut) (1.21.5)\n",
      "Requirement already satisfied: pyyaml>=5.1.2 in /opt/conda/lib/python3.9/site-packages (from attacut) (6.0)\n",
      "Requirement already satisfied: torch>=1.2.0 in /opt/conda/lib/python3.9/site-packages (from attacut) (2.0.0)\n",
      "Collecting nptyping>=0.2.0\n",
      "  Downloading nptyping-2.5.0-py3-none-any.whl (37 kB)\n",
      "Collecting fire>=0.1.3\n",
      "  Downloading fire-0.5.0.tar.gz (88 kB)\n",
      "     |████████████████████████████████| 88 kB 3.7 MB/s             \n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hRequirement already satisfied: six>=1.12.0 in /opt/conda/lib/python3.9/site-packages (from attacut) (1.16.0)\n",
      "Collecting docopt>=0.6.2\n",
      "  Downloading docopt-0.6.2.tar.gz (25 kB)\n",
      "  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hCollecting ssg>=0.0.4\n",
      "  Downloading ssg-0.0.8-py3-none-any.whl (473 kB)\n",
      "     |████████████████████████████████| 473 kB 17.1 MB/s            \n",
      "\u001b[?25hCollecting termcolor\n",
      "  Downloading termcolor-2.3.0-py3-none-any.whl (6.9 kB)\n",
      "Requirement already satisfied: typing-extensions<5.0.0,>=4.0.0 in /opt/conda/lib/python3.9/site-packages (from nptyping>=0.2.0->attacut) (4.5.0)\n",
      "Requirement already satisfied: tqdm>=4.32.2 in /opt/conda/lib/python3.9/site-packages (from ssg>=0.0.4->attacut) (4.62.3)\n",
      "Collecting python-crfsuite>=0.9.6\n",
      "  Downloading python_crfsuite-0.9.9-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.0 MB)\n",
      "     |████████████████████████████████| 1.0 MB 25.2 MB/s            \n",
      "\u001b[?25hRequirement already satisfied: nvidia-cusolver-cu11==11.4.0.1 in /opt/conda/lib/python3.9/site-packages (from torch>=1.2.0->attacut) (11.4.0.1)\n",
      "Requirement already satisfied: sympy in /opt/conda/lib/python3.9/site-packages (from torch>=1.2.0->attacut) (1.9)\n",
      "Requirement already satisfied: jinja2 in /opt/conda/lib/python3.9/site-packages (from torch>=1.2.0->attacut) (3.0.3)\n",
      "Requirement already satisfied: nvidia-cuda-runtime-cu11==11.7.99 in /opt/conda/lib/python3.9/site-packages (from torch>=1.2.0->attacut) (11.7.99)\n",
      "Requirement already satisfied: nvidia-cuda-nvrtc-cu11==11.7.99 in /opt/conda/lib/python3.9/site-packages (from torch>=1.2.0->attacut) (11.7.99)\n",
      "Requirement already satisfied: nvidia-cusparse-cu11==11.7.4.91 in /opt/conda/lib/python3.9/site-packages (from torch>=1.2.0->attacut) (11.7.4.91)\n",
      "Requirement already satisfied: nvidia-cuda-cupti-cu11==11.7.101 in /opt/conda/lib/python3.9/site-packages (from torch>=1.2.0->attacut) (11.7.101)\n",
      "Requirement already satisfied: nvidia-nvtx-cu11==11.7.91 in /opt/conda/lib/python3.9/site-packages (from torch>=1.2.0->attacut) (11.7.91)\n",
      "Requirement already satisfied: nvidia-cublas-cu11==11.10.3.66 in /opt/conda/lib/python3.9/site-packages (from torch>=1.2.0->attacut) (11.10.3.66)\n",
      "Requirement already satisfied: triton==2.0.0 in /opt/conda/lib/python3.9/site-packages (from torch>=1.2.0->attacut) (2.0.0)\n",
      "Requirement already satisfied: nvidia-curand-cu11==10.2.10.91 in /opt/conda/lib/python3.9/site-packages (from torch>=1.2.0->attacut) (10.2.10.91)\n",
      "Requirement already satisfied: filelock in /opt/conda/lib/python3.9/site-packages (from torch>=1.2.0->attacut) (3.12.0)\n",
      "Requirement already satisfied: nvidia-cudnn-cu11==8.5.0.96 in /opt/conda/lib/python3.9/site-packages (from torch>=1.2.0->attacut) (8.5.0.96)\n",
      "Requirement already satisfied: networkx in /opt/conda/lib/python3.9/site-packages (from torch>=1.2.0->attacut) (2.6.3)\n",
      "Requirement already satisfied: nvidia-nccl-cu11==2.14.3 in /opt/conda/lib/python3.9/site-packages (from torch>=1.2.0->attacut) (2.14.3)\n",
      "Requirement already satisfied: nvidia-cufft-cu11==10.9.0.58 in /opt/conda/lib/python3.9/site-packages (from torch>=1.2.0->attacut) (10.9.0.58)\n",
      "Requirement already satisfied: wheel in /opt/conda/lib/python3.9/site-packages (from nvidia-cublas-cu11==11.10.3.66->torch>=1.2.0->attacut) (0.37.1)\n",
      "Requirement already satisfied: setuptools in /opt/conda/lib/python3.9/site-packages (from nvidia-cublas-cu11==11.10.3.66->torch>=1.2.0->attacut) (59.8.0)\n",
      "Requirement already satisfied: lit in /opt/conda/lib/python3.9/site-packages (from triton==2.0.0->torch>=1.2.0->attacut) (16.0.2)\n",
      "Requirement already satisfied: cmake in /opt/conda/lib/python3.9/site-packages (from triton==2.0.0->torch>=1.2.0->attacut) (3.26.3)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.9/site-packages (from jinja2->torch>=1.2.0->attacut) (2.0.1)\n",
      "Requirement already satisfied: mpmath>=0.19 in /opt/conda/lib/python3.9/site-packages (from sympy->torch>=1.2.0->attacut) (1.2.1)\n",
      "Building wheels for collected packages: docopt, fire\n",
      "  Building wheel for docopt (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for docopt: filename=docopt-0.6.2-py2.py3-none-any.whl size=13723 sha256=2d1316824a3d1906d58031a4799094542e0ef7f3deda6248ad21e6b7178e19c9\n",
      "  Stored in directory: /home/st123028/.cache/pip/wheels/70/4a/46/1309fc853b8d395e60bafaf1b6df7845bdd82c95fd59dd8d2b\n",
      "  Building wheel for fire (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for fire: filename=fire-0.5.0-py2.py3-none-any.whl size=116951 sha256=234c72cd5919f90c02ae8911d260607aac30680419432be2e1c2921af8c963c4\n",
      "  Stored in directory: /home/st123028/.cache/pip/wheels/f7/f1/89/b9ea2bf8f80ec027a88fef1d354b3816b4d3d29530988972f6\n",
      "Successfully built docopt fire\n",
      "Installing collected packages: termcolor, python-crfsuite, fire, ssg, nptyping, docopt, attacut\n",
      "Successfully installed attacut-1.0.6 docopt-0.6.2 fire-0.5.0 nptyping-2.5.0 python-crfsuite-0.9.9 ssg-0.0.8 termcolor-2.3.0\n"
     ]
    }
   ],
   "source": [
    "!pip install attacut"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "executionInfo": {
     "elapsed": 181,
     "status": "aborted",
     "timestamp": 1682613921787,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "bpxkW45RVQwa"
   },
   "outputs": [],
   "source": [
    "from attacut import tokenize, Tokenizer\n",
    "from torchtext.data.utils import get_tokenizer\n",
    "token_transform[SRC_LANGUAGE] = Tokenizer(model=\"attacut-sc\")\n",
    "token_transform[TRG_LANGUAGE] = get_tokenizer('spacy', language='en_core_web_sm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "executionInfo": {
     "elapsed": 182,
     "status": "aborted",
     "timestamp": 1682613921788,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "mjqsaQzGVc0M"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sentence:  ฉันรักมันได้ปกป้องโทรศัพท์ของฉันค่อนข้างดี\n",
      "Tokenization:  ['ฉัน', 'รัก', 'มัน', 'ได้', 'ปกป้อง', 'โทรศัพท์', 'ของ', 'ฉัน', 'ค่อนข้าง', 'ดี']\n"
     ]
    }
   ],
   "source": [
    "#example of tokenization of the thai part\n",
    "print(\"Sentence: \", train[0][1])\n",
    "print(\"Tokenization: \", token_transform[SRC_LANGUAGE].tokenize(train[0][1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "executionInfo": {
     "elapsed": 216,
     "status": "aborted",
     "timestamp": 1682613921823,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "avRf44mqVizF"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sentence:  I love it, has protected my phone quite well.\n",
      "Tokenization:  ['I', ' ', 'love', ' ', 'it', ',', ' ', 'has', ' ', 'protected', ' ', 'my', ' ', 'phone', ' ', 'quite', ' ', 'well', '.']\n"
     ]
    }
   ],
   "source": [
    "#example of tokenization of the english part\n",
    "print(\"Sentence: \", train[0][0])\n",
    "print(\"Tokenization: \", token_transform[SRC_LANGUAGE].tokenize(train[0][0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "executionInfo": {
     "elapsed": 217,
     "status": "aborted",
     "timestamp": 1682613921824,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "aQIzFuehVxeH"
   },
   "outputs": [],
   "source": [
    "# helper function to yield list of tokens\n",
    "# here data can be `train` or `val` or `test`\n",
    "def yield_tokens(data, language):\n",
    "    language_index = {SRC_LANGUAGE: 0, TRG_LANGUAGE: 1}\n",
    "\n",
    "    for data_sample in data:\n",
    "        if language == SRC_LANGUAGE:\n",
    "            yield token_transform[language].tokenize(data_sample[language_index[language]])\n",
    "        elif language == TRG_LANGUAGE:\n",
    "            yield token_transform[language](data_sample[language_index[language]])\n",
    "\n",
    "# Define special symbols and indices\n",
    "UNK_IDX, PAD_IDX, SOS_IDX, EOS_IDX = 0, 1, 2, 3\n",
    "# Make sure the tokens are in order of their indices to properly insert them in vocab\n",
    "special_symbols = ['<unk>', '<pad>', '<sos>', '<eos>']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "executionInfo": {
     "elapsed": 216,
     "status": "aborted",
     "timestamp": 1682613921824,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "YNp1-pH5V-1e"
   },
   "outputs": [],
   "source": [
    "from torchtext.vocab import build_vocab_from_iterator\n",
    "\n",
    "for ln in [SRC_LANGUAGE, TRG_LANGUAGE]:\n",
    "    # Create torchtext's Vocab object \n",
    "    vocab_transform[ln] = build_vocab_from_iterator(yield_tokens(train, ln), \n",
    "                                                    min_freq=2,   #if not, everything will be treated as UNK\n",
    "                                                    specials=special_symbols,\n",
    "                                                    special_first=True) #indicates whether to insert symbols at the beginning or at the end                                            \n",
    "# Set UNK_IDX as the default index. This index is returned when the token is not found. \n",
    "# If not set, it throws RuntimeError when the queried token is not found in the Vocabulary. \n",
    "for ln in [SRC_LANGUAGE, TRG_LANGUAGE]:\n",
    "    vocab_transform[ln].set_default_index(UNK_IDX)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "executionInfo": {
     "elapsed": 217,
     "status": "aborted",
     "timestamp": 1682613921825,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "rFef4Q4eWAoa"
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "with open('vocab.pkl', 'wb') as file:\n",
    "      \n",
    "    # A new file will be created\n",
    "    pickle.dump(vocab_transform, file)\n",
    "import pickle\n",
    "with open('thai_vocab.pkl', 'wb') as file:\n",
    "      \n",
    "    # A new file will be created\n",
    "    pickle.dump(vocab_transform[TRG_LANGUAGE], file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "executionInfo": {
     "elapsed": 217,
     "status": "aborted",
     "timestamp": 1682613921825,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "N8bJn-BPWDEp"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[409, 17, 12, 0, 12]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab_transform[SRC_LANGUAGE](['here', 'is', 'a', 'unknownword', 'a'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "executionInfo": {
     "elapsed": 217,
     "status": "aborted",
     "timestamp": 1682613921826,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "pKTmnyZfWGet"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, 10954, 0]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab_transform[TRG_LANGUAGE](['สยฎ', 'ทดสอบ', 'คำภาษาไทย'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "executionInfo": {
     "elapsed": 219,
     "status": "aborted",
     "timestamp": 1682613921828,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "o7n5jFahWR4x"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'ประกอบด้วย'"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#we can reverse its English....\n",
    "mapping = vocab_transform[TRG_LANGUAGE].get_itos()\n",
    "# #print 1816, for example\n",
    "mapping[400]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "executionInfo": {
     "elapsed": 219,
     "status": "aborted",
     "timestamp": 1682613921828,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "KuXhzSxsWVnu"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Machine'"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#we can reverse its Thailand....\n",
    "mapping = vocab_transform[SRC_LANGUAGE].get_itos()\n",
    "# #print 1816, for example\n",
    "mapping[400]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "executionInfo": {
     "elapsed": 220,
     "status": "aborted",
     "timestamp": 1682613921829,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "nXP3GzemWaQn"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('<pad>', '<sos>', '<eos>')"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mapping[1], mapping[2], mapping[3]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "E6NMcCyXWdtt"
   },
   "source": [
    "# 4. Preparing dataloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "executionInfo": {
     "elapsed": 221,
     "status": "aborted",
     "timestamp": 1682613921830,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "KBwOah7KWdYe"
   },
   "outputs": [],
   "source": [
    "from torch.nn.utils.rnn import pad_sequence\n",
    "from torch.utils.data import DataLoader\n",
    "import multiprocessing as mp\n",
    "BATCH_SIZE = 64\n",
    "\n",
    "# helper function to club together sequential operations\n",
    "def sequential_transforms(*transforms):\n",
    "    global func\n",
    "    def func(txt_input):\n",
    "        for transform in transforms:\n",
    "            if transform == token_transform[SRC_LANGUAGE]:\n",
    "                txt_input = transform.tokenize(txt_input)\n",
    "            else:\n",
    "                txt_input = transform(txt_input)\n",
    "        return txt_input\n",
    "\n",
    "    return func\n",
    "\n",
    "# function to add BOS/EOS and create tensor for input sequence indices\n",
    "def tensor_transform(token_ids):\n",
    "    return torch.cat((torch.tensor([SOS_IDX]), \n",
    "                      torch.tensor(token_ids), \n",
    "                      torch.tensor([EOS_IDX])))\n",
    "\n",
    "# src and trg language text transforms to convert raw strings into tensors indices\n",
    "text_transform = {}\n",
    "for ln in [SRC_LANGUAGE, TRG_LANGUAGE]:\n",
    "    text_transform[ln] = sequential_transforms(token_transform[ln], #Tokenization\n",
    "                                               vocab_transform[ln], #Numericalization\n",
    "                                               tensor_transform) # Add BOS/EOS and create tensor\n",
    "\n",
    "# function to collate data samples into batch tesors\n",
    "def collate_batch(batch):\n",
    "    src_batch, src_len_batch, trg_batch = [], [], []\n",
    "    for src_sample, trg_sample in batch:\n",
    "        processed_text = text_transform[SRC_LANGUAGE](src_sample.rstrip(\"\\n\"))\n",
    "        src_batch.append(processed_text)\n",
    "        trg_batch.append(text_transform[TRG_LANGUAGE](trg_sample.rstrip(\"\\n\")))\n",
    "        src_len_batch.append(processed_text.size(0))\n",
    "\n",
    "    src_batch = pad_sequence(src_batch, padding_value=PAD_IDX)\n",
    "    trg_batch = pad_sequence(trg_batch, padding_value=PAD_IDX)\n",
    "    import pickle\n",
    "        \n",
    "    return src_batch, torch.tensor(src_len_batch, dtype=torch.int64), trg_batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "executionInfo": {
     "elapsed": 221,
     "status": "aborted",
     "timestamp": 1682613921830,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "7GGswGXIWc6n"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'th': <function __main__.func(txt_input)>,\n",
       " 'en': <function __main__.func(txt_input)>}"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_transform"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "executionInfo": {
     "elapsed": 234,
     "status": "aborted",
     "timestamp": 1682613921844,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "cShLToo3WnFb"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Thai shape:  torch.Size([276, 32])\n",
      "English shape:  torch.Size([61, 32])\n"
     ]
    }
   ],
   "source": [
    "batch_size = 32\n",
    "\n",
    "train_loader = DataLoader(train, batch_size=batch_size,\n",
    "                              shuffle=True, collate_fn=collate_batch)\n",
    "valid_loader = DataLoader(val, batch_size=batch_size,\n",
    "                              shuffle=True, collate_fn=collate_batch)\n",
    "test_loader  = DataLoader(test, batch_size=batch_size,\n",
    "                             shuffle=True, collate_fn=collate_batch)\n",
    "\n",
    "for th, _, en in train_loader:\n",
    "    break\n",
    "\n",
    "print(\"Thai shape: \", th.shape)  # (seq len, batch_size)\n",
    "print(\"English shape: \", en.shape)   # (seq len, batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "executionInfo": {
     "elapsed": 235,
     "status": "aborted",
     "timestamp": 1682613921845,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "B6BGYj47WywN"
   },
   "outputs": [],
   "source": [
    "# train_loader_length = len(list(iter(train_loader)))\n",
    "# val_loader_length   = len(list(iter(valid_loader)))\n",
    "# test_loader_length  = len(list(iter(test_loader)))\n",
    "# print(train_loader_length)\n",
    "# print(val_loader_length)\n",
    "# print(test_loader_length)\n",
    "# # 2536\n",
    "# # 282\n",
    "# # 314"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "executionInfo": {
     "elapsed": 235,
     "status": "aborted",
     "timestamp": 1682613921845,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "UmCswV56W0XC"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2536, 282, 314)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_loader_length = 2536\n",
    "val_loader_length =282\n",
    "test_loader_length = 314\n",
    "\n",
    "train_loader_length, val_loader_length, test_loader_length  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "_ptNUxSvW5Am"
   },
   "source": [
    "# 5. Model Designing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "executionInfo": {
     "elapsed": 236,
     "status": "aborted",
     "timestamp": 1682613921846,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "GJRfvUdBW2pY"
   },
   "outputs": [],
   "source": [
    "class Encoder(nn.Module):\n",
    "    def __init__(self, input_dim, emb_dim, hid_dim, dropout):\n",
    "        super().__init__()\n",
    "\n",
    "        self.embedding = nn.Embedding(input_dim, emb_dim)\n",
    "        self.rnn       = nn.GRU(emb_dim, hid_dim, bidirectional=True)\n",
    "        self.fc        = nn.Linear(hid_dim * 2, hid_dim)\n",
    "        self.dropout   = nn.Dropout(dropout)\n",
    "        \n",
    "    def forward(self, src, src_len):\n",
    "        \n",
    "        #src: [src len, batch size]\n",
    "        #src len: [batch size]\n",
    "        \n",
    "        embedded = self.dropout(self.embedding(src))\n",
    "        \n",
    "        packed_embedded = nn.utils.rnn.pack_padded_sequence(embedded, \n",
    "                            src_len.to('cpu'), enforce_sorted=False)\n",
    "        \n",
    "        #packed_outputs contain all hidden states including padding guy\n",
    "        #hidden contains the last hidden states of the non-padded guys\n",
    "        packed_outputs, hidden = self.rnn(packed_embedded)\n",
    "        #hidden: [n layers * num directions, batch size, hid dim]\n",
    "        \n",
    "        #convert packed_outputs to the guy that does not contain hidden states for padding\n",
    "        outputs, _ = nn.utils.rnn.pad_packed_sequence(packed_outputs)\n",
    "        #outputs: [src len, batch size, hid dim * num directions]\n",
    "        \n",
    "        #take the last hidden states from backward and forward\n",
    "        #hidden: (f, b, f, b)\n",
    "        forward  = hidden[-2, :, :]  #[batch size, hid dim]\n",
    "        backward = hidden[-1, :, :]  #[batch size, hid dim]\n",
    "        \n",
    "        hidden = torch.tanh(self.fc(torch.cat((forward, backward), dim = 1))) \n",
    "        #hidden: [batch size, hid dim]\n",
    "        \n",
    "        return outputs, hidden"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "executionInfo": {
     "elapsed": 237,
     "status": "aborted",
     "timestamp": 1682613921847,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "7qhUTnQGW9wB"
   },
   "outputs": [],
   "source": [
    "class Attention(nn.Module):\n",
    "    def __init__(self, hid_dim, variants):\n",
    "        super().__init__()\n",
    "        self.variants = variants\n",
    "        self.v = nn.Linear(hid_dim, 1, bias = False)\n",
    "        self.W = nn.Linear(hid_dim,     hid_dim) #for decoder\n",
    "        self.U = nn.Linear(hid_dim * 2, hid_dim) #for encoder outputs\n",
    "                \n",
    "    def forward(self, hidden, encoder_outputs, mask):\n",
    "        \n",
    "        #hidden = [batch size, hid dim]\n",
    "        #encoder_outputs = [src len, batch size, hid dim * 2]\n",
    "        \n",
    "        batch_size = encoder_outputs.shape[1]\n",
    "        src_len = encoder_outputs.shape[0]\n",
    "\n",
    "        encoder_outputs = encoder_outputs.permute(1, 0, 2)\n",
    "        #encoder_outputs = [batch size, src len, hid dim * 2]\n",
    "\n",
    "        if self.variants == 'additive': #work\n",
    "            #repeat decoder hidden state src_len times\n",
    "            hidden = hidden.unsqueeze(1).repeat(1, src_len, 1)\n",
    "            #hidden = [batch size, src len, hid dim]\n",
    "            \n",
    "            energy = torch.tanh(self.W(hidden) + self.U(encoder_outputs))\n",
    "            #energy = [batch size, src len, hid dim]\n",
    "            \n",
    "            attention = self.v(energy).squeeze(2)\n",
    "            #attention = [batch size, src len]\n",
    "            \n",
    "        elif self.variants == 'general': #work\n",
    "            hidden = hidden.unsqueeze(1).repeat(1, 1, 2)\n",
    "            #hidden = [batch size, 1, hid dim*2]\n",
    "            #encoder_outputs = [batch size, hid dim * 2, src len]\n",
    "\n",
    "            energy = torch.bmm(hidden, encoder_outputs.transpose(1, 2))\n",
    "            attention = energy.squeeze(1)\n",
    "            #attention = [batch size, src len]\n",
    "\n",
    "        elif self.variants == 'multiplicative':\n",
    "            wh = self.W(hidden).unsqueeze(1).repeat(1, 1, 2)\n",
    "            #wh = [batch size, 1, hid dim*2]\n",
    "            #encoder_outputs = [batch size, hid dim * 2, src len]\n",
    "\n",
    "            energy = torch.bmm(wh, encoder_outputs.transpose(1, 2))\n",
    "            attention = energy.squeeze(1)\n",
    "\n",
    "        #use masked_fill_ if you want in-place\n",
    "        attention = attention.masked_fill(mask, -1e10)\n",
    "        #attention = [batch size, src len]\n",
    "        return F.softmax(attention, dim = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "executionInfo": {
     "elapsed": 237,
     "status": "aborted",
     "timestamp": 1682613921848,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "mszTf8QnXAXV"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[           9, -10000000000,            7,            2, -10000000000,\n",
      "         -10000000000],\n",
      "        [          99, -10000000000, -10000000000,            0,            8,\n",
      "                    9]])\n"
     ]
    }
   ],
   "source": [
    "#example of masked_fill\n",
    "#reall that 1 is pad_idx\n",
    "x = torch.tensor([ [9, 1, 7, 2, 1, 1], [99, 1, 1, 0, 8, 9] ])\n",
    "\n",
    "mask = (x == PAD_IDX)\n",
    "\n",
    "x.masked_fill_(mask, -1e10)\n",
    "print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "executionInfo": {
     "elapsed": 238,
     "status": "aborted",
     "timestamp": 1682613921849,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "vxVZvLfMXClp"
   },
   "outputs": [],
   "source": [
    "class Decoder(nn.Module):\n",
    "    def __init__(self, output_dim, emb_dim, hid_dim, dropout, attention):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.output_dim = output_dim\n",
    "        self.attention = attention\n",
    "        self.embedding = nn.Embedding(output_dim, emb_dim)\n",
    "        self.gru = nn.GRU((hid_dim * 2) + emb_dim, hid_dim)\n",
    "        self.fc = nn.Linear((hid_dim * 2) + hid_dim + emb_dim, output_dim)\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "        \n",
    "    def forward(self, input, hidden, encoder_outputs, mask):\n",
    "        \n",
    "        #input = [batch size]\n",
    "        #hidden = [batch size, hid dim]\n",
    "        #encoder_outputs = [src len, batch size, hid dim * 2]\n",
    "        #mask = [batch size, src len]\n",
    "        \n",
    "        input = input.unsqueeze(0)\n",
    "        #input = [1, batch size]\n",
    "        \n",
    "        embedded = self.dropout(self.embedding(input))\n",
    "        #embedded = [1, batch size, emb dim]\n",
    "        \n",
    "        a = self.attention(hidden, encoder_outputs, mask)\n",
    "        #a = [batch size, src len]\n",
    "\n",
    "        a = a.unsqueeze(1)\n",
    "        #a = [batch size, 1, src len]\n",
    "        \n",
    "        encoder_outputs = encoder_outputs.permute(1, 0, 2)\n",
    "        #encoder_outputs = [batch size, src len, hid dim * 2]\n",
    "        \n",
    "        weighted = torch.bmm(a, encoder_outputs) #Ct\n",
    "        #weighted = [batch size, 1, hid dim * 2]\n",
    "        \n",
    "        weighted = weighted.permute(1, 0, 2)\n",
    "        #weighted = [1, batch size, hid dim * 2]\n",
    "        \n",
    "        rnn_input = torch.cat((embedded, weighted), dim = 2)\n",
    "        #rnn_input = [1, batch size, (hid dim * 2) + emb dim]\n",
    "        \n",
    "        output, hidden = self.gru(rnn_input, hidden.unsqueeze(0))\n",
    "        #output = [seq len, batch size, dec hid dim * n directions]\n",
    "        #hidden = [1, batch size, hid dim]\n",
    "        assert (output == hidden).all()\n",
    "\n",
    "        embedded = embedded.squeeze(0)\n",
    "        output   = output.squeeze(0)\n",
    "        weighted = weighted.squeeze(0)\n",
    "        \n",
    "        prediction = self.fc(torch.cat((output, weighted, embedded), dim = 1))\n",
    "        #prediction = [batch size, output dim]\n",
    "        \n",
    "        return prediction, hidden.squeeze(0), a.squeeze(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "executionInfo": {
     "elapsed": 238,
     "status": "aborted",
     "timestamp": 1682613921849,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "HgxyRZHJXFzJ"
   },
   "outputs": [],
   "source": [
    "class Seq2SeqPackedAttention(nn.Module):\n",
    "    def __init__(self, encoder, decoder, src_pad_idx, device):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.encoder = encoder\n",
    "        self.decoder = decoder\n",
    "        self.src_pad_idx = src_pad_idx\n",
    "        self.device = device\n",
    "        \n",
    "    def create_mask(self, src):\n",
    "        #src: [src len, batch size]\n",
    "        \n",
    "        mask = (src == self.src_pad_idx).permute(1, 0)\n",
    "        #mask: [batch size, src len]\n",
    "        #we need to permute to make the mask same shape as attention...\n",
    "        return mask\n",
    "\n",
    "        \n",
    "    def forward(self, src, src_len, trg, teacher_forcing_ratio = 0.5):\n",
    "        \n",
    "        #src: [src len, batch size]\n",
    "        #src len: [batch size]\n",
    "        #trg: [trg len, batch size]\n",
    "        \n",
    "        batch_size = src.shape[1]\n",
    "        trg_len    = trg.shape[0]\n",
    "        trg_vocab_size = self.decoder.output_dim #define in decoder\n",
    "        \n",
    "        #because decoder decodes each step....let's create a list that gonna append the result to this guy\n",
    "        outputs = torch.zeros(trg_len, batch_size, trg_vocab_size).to(self.device)\n",
    "        \n",
    "        #because decoder decodes each step....let's memorize the attention done in each step....\n",
    "        attentions = torch.zeros(trg_len, batch_size, src.shape[0]).to(self.device)\n",
    "        \n",
    "        #let's start!!!\n",
    "        #1. encoder\n",
    "        encoder_outputs, hidden = self.encoder(src, src_len)\n",
    "        #encoder_outputs: [src len, batch size, hid dim * num directions]\n",
    "        #hidden: [batch size, hid dim]\n",
    "        \n",
    "        #set the first input to the decoder\n",
    "        input_ = trg[0,:]  #basically <sos>\n",
    "        \n",
    "        #create the mask for use in this step\n",
    "        mask = self.create_mask(src)\n",
    "        \n",
    "        #2. for each of trg word\n",
    "        for t in range(1, trg_len):\n",
    "\n",
    "            #3. decode (hidden is always carry forward)\n",
    "            output, hidden, attention = self.decoder(input_, hidden, encoder_outputs, mask)\n",
    "            #output:   [batch size, output_dim]\n",
    "            #hidden:   [batch size, hid_dim]\n",
    "            #attention::[batch size, src len]  ==> how each of src token is important to input_ \n",
    "            \n",
    "            #4. append the results to outputs and attentions\n",
    "            outputs[t] = output\n",
    "            attentions[t] = attention\n",
    "            \n",
    "            #5. get the result, using argmax\n",
    "            top1 = output.argmax(1)  #find the maximum index of dimension 1, which is output_dim\n",
    "            \n",
    "            #6. based on the teacher forcing ratio, \n",
    "            teacher_force_or_not = random.random() < teacher_forcing_ratio\n",
    "                    #if teacher forcing, next input is the next trg\n",
    "                    #if no teacher forcing, the next input is the argmax guy...\n",
    "            input_ = trg[t] if teacher_force_or_not else top1  #autoregressive\n",
    "            \n",
    "        return outputs, attentions #outputs for predicting the word, attentions to see which word is important"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "executionInfo": {
     "elapsed": 239,
     "status": "aborted",
     "timestamp": 1682613921850,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "B-I17fI8XIqy"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Seq2SeqPackedAttention(\n",
       "  (encoder): Encoder(\n",
       "    (embedding): Embedding(44036, 128)\n",
       "    (rnn): GRU(128, 256, bidirectional=True)\n",
       "    (fc): Linear(in_features=512, out_features=256, bias=True)\n",
       "    (dropout): Dropout(p=0.5, inplace=False)\n",
       "  )\n",
       "  (decoder): Decoder(\n",
       "    (attention): Attention(\n",
       "      (v): Linear(in_features=256, out_features=1, bias=False)\n",
       "      (W): Linear(in_features=256, out_features=256, bias=True)\n",
       "      (U): Linear(in_features=512, out_features=256, bias=True)\n",
       "    )\n",
       "    (embedding): Embedding(35669, 128)\n",
       "    (gru): GRU(640, 256)\n",
       "    (fc): Linear(in_features=896, out_features=35669, bias=True)\n",
       "    (dropout): Dropout(p=0.5, inplace=False)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def initialize_weights(m):\n",
    "    for name, param in m.named_parameters():\n",
    "        if 'weight' in name:\n",
    "            nn.init.normal_(param.data, mean=0, std=0.01)\n",
    "        else:\n",
    "            nn.init.constant_(param.data, 0)\n",
    "\n",
    "\n",
    "input_dim   = len(vocab_transform[SRC_LANGUAGE])\n",
    "output_dim  = len(vocab_transform[TRG_LANGUAGE])\n",
    "emb_dim     = 128  \n",
    "hid_dim     = 256  \n",
    "dropout     = 0.5\n",
    "SRC_PAD_IDX = PAD_IDX\n",
    "\n",
    "attn = Attention(hid_dim, variants='additive')\n",
    "enc  = Encoder(input_dim,  emb_dim,  hid_dim, dropout)\n",
    "dec  = Decoder(output_dim, emb_dim,  hid_dim, dropout, attn)\n",
    "\n",
    "model_additive = Seq2SeqPackedAttention(enc, dec, SRC_PAD_IDX, device).to(device)\n",
    "model_additive.apply(initialize_weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "executionInfo": {
     "elapsed": 240,
     "status": "aborted",
     "timestamp": 1682613921851,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "O44O-uYVXMGU"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5636608\n",
      " 98304\n",
      "196608\n",
      "   768\n",
      "   768\n",
      " 98304\n",
      "196608\n",
      "   768\n",
      "   768\n",
      "131072\n",
      "   256\n",
      "   256\n",
      " 65536\n",
      "   256\n",
      "131072\n",
      "   256\n",
      "4565632\n",
      "491520\n",
      "196608\n",
      "   768\n",
      "   768\n",
      "31959424\n",
      " 35669\n",
      "______\n",
      "43808597\n"
     ]
    }
   ],
   "source": [
    "def count_parameters(model):\n",
    "    params = [p.numel() for p in model.parameters() if p.requires_grad]\n",
    "    for item in params:\n",
    "        print(f'{item:>6}')\n",
    "    print(f'______\\n{sum(params):>6}')\n",
    "    \n",
    "count_parameters(model_additive)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "0MNq_Rg2XQdH"
   },
   "source": [
    "# 6. Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "executionInfo": {
     "elapsed": 243,
     "status": "aborted",
     "timestamp": 1682613921854,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "-yKj_eeEXPss"
   },
   "outputs": [],
   "source": [
    "import torch.optim as optim\n",
    "\n",
    "lr = 0.001\n",
    "\n",
    "#training hyperparameters\n",
    "optimizer = optim.Adam(model_additive.parameters(), lr=lr)\n",
    "criterion = nn.CrossEntropyLoss(ignore_index = PAD_IDX)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "executionInfo": {
     "elapsed": 243,
     "status": "aborted",
     "timestamp": 1682613921855,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "Aj064e8ZXZCt"
   },
   "outputs": [],
   "source": [
    "def train(model, loader, optimizer, criterion, clip, loader_length):\n",
    "    \n",
    "    model.train()\n",
    "    epoch_loss = 0\n",
    "    \n",
    "    for src, src_length, trg in loader:\n",
    "        \n",
    "        src = src.to(device)\n",
    "        trg = trg.to(device)\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        output, attentions = model(src, src_length, trg)\n",
    "        \n",
    "        #trg    = [trg len, batch size]\n",
    "        #output = [trg len, batch size, output dim]\n",
    "        output_dim = output.shape[-1]\n",
    "        \n",
    "        #the loss function only works on 2d inputs with 1d targets thus we need to flatten each of them\n",
    "        output = output[1:].view(-1, output_dim)\n",
    "        trg    = trg[1:].view(-1)\n",
    "        #trg    = [(trg len - 1) * batch size]\n",
    "        #output = [(trg len - 1) * batch size, output dim]\n",
    "        \n",
    "        loss = criterion(output, trg)\n",
    "        \n",
    "        loss.backward()\n",
    "        \n",
    "        #clip the gradients to prevent them from exploding (a common issue in RNNs)\n",
    "        torch.nn.utils.clip_grad_norm_(model.parameters(), clip)\n",
    "        \n",
    "        optimizer.step()\n",
    "        \n",
    "        epoch_loss += loss.item()\n",
    "        \n",
    "    return epoch_loss / loader_length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "executionInfo": {
     "elapsed": 244,
     "status": "aborted",
     "timestamp": 1682613921856,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "CK6JE9GnXbCF"
   },
   "outputs": [],
   "source": [
    "def evaluate(model, loader, criterion, loader_length):\n",
    "        \n",
    "    #turn off dropout (and batch norm if used)\n",
    "    model.eval()\n",
    "    \n",
    "    epoch_loss = 0\n",
    "    \n",
    "    with torch.no_grad():\n",
    "    \n",
    "        for src, src_length, trg in loader:\n",
    "        \n",
    "            src = src.to(device)\n",
    "            trg = trg.to(device)\n",
    "\n",
    "            output, attentions = model(src, src_length, trg, 0) #turn off teacher forcing\n",
    "\n",
    "            #trg    = [trg len, batch size]\n",
    "            #output = [trg len, batch size, output dim]\n",
    "\n",
    "            output_dim = output.shape[-1]\n",
    "            \n",
    "            output = output[1:].view(-1, output_dim)\n",
    "            trg = trg[1:].view(-1)\n",
    "\n",
    "            #trg    = [(trg len - 1) * batch size]\n",
    "            #output = [(trg len - 1) * batch size, output dim]\n",
    "\n",
    "            loss = criterion(output, trg)\n",
    "            \n",
    "            epoch_loss += loss.item()\n",
    "        \n",
    "    return epoch_loss / loader_length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "executionInfo": {
     "elapsed": 245,
     "status": "aborted",
     "timestamp": 1682613921857,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "m-PHa0U7Xcw2"
   },
   "outputs": [],
   "source": [
    "def epoch_time(start_time, end_time):\n",
    "    elapsed_time = end_time - start_time\n",
    "    elapsed_mins = int(elapsed_time / 60)\n",
    "    elapsed_secs = int(elapsed_time - (elapsed_mins * 60))\n",
    "    return elapsed_mins, elapsed_secs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "executionInfo": {
     "elapsed": 246,
     "status": "aborted",
     "timestamp": 1682613921858,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "wHeu-JL-XeLN"
   },
   "outputs": [
    {
     "ename": "OutOfMemoryError",
     "evalue": "CUDA out of memory. Tried to allocate 458.00 MiB (GPU 0; 10.76 GiB total capacity; 8.41 GiB already allocated; 220.69 MiB free; 9.65 GiB reserved in total by PyTorch) If reserved memory is >> allocated memory try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mOutOfMemoryError\u001b[0m                          Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_180/649824707.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     12\u001b[0m     \u001b[0mstart_time\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 14\u001b[0;31m     \u001b[0mtrain_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_additive\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_loader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclip\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_loader_length\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     15\u001b[0m     \u001b[0mvalid_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_additive\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalid_loader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_loader_length\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/tmp/ipykernel_180/1330848242.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(model, loader, optimizer, criterion, clip, loader_length)\u001b[0m\n\u001b[1;32m     23\u001b[0m         \u001b[0;31m#output = [(trg len - 1) * batch size, output dim]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 25\u001b[0;31m         \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     26\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     27\u001b[0m         \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.9/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1499\u001b[0m                 \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_pre_hooks\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_hooks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1500\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1502\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1503\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.9/site-packages/torch/nn/modules/loss.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input, target)\u001b[0m\n\u001b[1;32m   1172\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1173\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1174\u001b[0;31m         return F.cross_entropy(input, target, weight=self.weight,\n\u001b[0m\u001b[1;32m   1175\u001b[0m                                \u001b[0mignore_index\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mignore_index\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreduction\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreduction\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1176\u001b[0m                                label_smoothing=self.label_smoothing)\n",
      "\u001b[0;32m/opt/conda/lib/python3.9/site-packages/torch/nn/functional.py\u001b[0m in \u001b[0;36mcross_entropy\u001b[0;34m(input, target, weight, size_average, ignore_index, reduce, reduction, label_smoothing)\u001b[0m\n\u001b[1;32m   3027\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0msize_average\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mreduce\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3028\u001b[0m         \u001b[0mreduction\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_Reduction\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlegacy_get_string\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msize_average\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreduce\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3029\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_C\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_nn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcross_entropy_loss\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_Reduction\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_enum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreduction\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mignore_index\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabel_smoothing\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3030\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3031\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mOutOfMemoryError\u001b[0m: CUDA out of memory. Tried to allocate 458.00 MiB (GPU 0; 10.76 GiB total capacity; 8.41 GiB already allocated; 220.69 MiB free; 9.65 GiB reserved in total by PyTorch) If reserved memory is >> allocated memory try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF"
     ]
    }
   ],
   "source": [
    "best_valid_loss = float('inf')\n",
    "num_epochs = 1\n",
    "clip       = 1\n",
    "\n",
    "save_path = f'fullSCB.pt'\n",
    "\n",
    "train_losses = []\n",
    "valid_losses = []\n",
    "\n",
    "for epoch in range(num_epochs):\n",
    "    \n",
    "    start_time = time.time()\n",
    "\n",
    "    train_loss = train(model_additive, train_loader, optimizer, criterion, clip, train_loader_length)\n",
    "    valid_loss = evaluate(model_additive, valid_loader, criterion, val_loader_length)\n",
    "    \n",
    "    #for plotting\n",
    "    train_losses.append(train_loss)\n",
    "    valid_losses.append(valid_loss)\n",
    "    \n",
    "    end_time = time.time()\n",
    "    \n",
    "    epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n",
    "\n",
    "    \n",
    "    if valid_loss < best_valid_loss:\n",
    "        best_valid_loss = valid_loss\n",
    "        torch.save(model_additive.state_dict(), save_path)\n",
    "    \n",
    "    print(f'Epoch: {epoch+1:02} | Time: {epoch_mins}m {epoch_secs}s')\n",
    "    print(f'\\tTrain Loss: {train_loss:.3f} | Train PPL: {math.exp(train_loss):7.3f}')\n",
    "    print(f'\\t Val. Loss: {valid_loss:.3f} |  Val. PPL: {math.exp(valid_loss):7.3f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 249,
     "status": "aborted",
     "timestamp": 1682613921861,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "N9zcKrerXgPZ"
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "fig = plt.figure(figsize=(5, 3))\n",
    "ax = fig.add_subplot(1, 1, 1)\n",
    "ax.plot(train_losses, label = 'train loss')\n",
    "ax.plot(valid_losses, label = 'valid loss')\n",
    "plt.legend()\n",
    "ax.set_xlabel('updates')\n",
    "ax.set_ylabel('loss')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 250,
     "status": "aborted",
     "timestamp": 1682613921862,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "ZDe4sL16Xh9-"
   },
   "outputs": [],
   "source": [
    "model_additive.load_state_dict(torch.load(save_path))\n",
    "test_loss = evaluate(model_additive, test_loader, criterion, test_loader_length)\n",
    "\n",
    "print(f'| Test Loss: {test_loss:.3f} | Test PPL: {math.exp(test_loss):7.3f} |')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "6TB4ui4QXkc2"
   },
   "source": [
    "# 7. Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 249,
     "status": "aborted",
     "timestamp": 1682613921862,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "kBuZa8dnXj18"
   },
   "outputs": [],
   "source": [
    "dataset[0][0], dataset[0][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 250,
     "status": "aborted",
     "timestamp": 1682613921863,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "e9pdo_AmXpM2"
   },
   "outputs": [],
   "source": [
    "src_text = text_transform[SRC_LANGUAGE](dataset[0][0]).to(device)\n",
    "trg_text = text_transform[TRG_LANGUAGE](dataset[0][1]).to(device)\n",
    "src_text, trg_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 255,
     "status": "aborted",
     "timestamp": 1682613921868,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "zpJvh67uXq6Z"
   },
   "outputs": [],
   "source": [
    "src_text = src_text.reshape(-1, 1)  #because batch_size is 1\n",
    "trg_text = trg_text.reshape(-1, 1)\n",
    "src_text.shape, trg_text.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 257,
     "status": "aborted",
     "timestamp": 1682613921870,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "_fPLZNchXsbi"
   },
   "outputs": [],
   "source": [
    "\n",
    "text_length = torch.tensor([src_text.size(0)]).to(dtype=torch.int64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 258,
     "status": "aborted",
     "timestamp": 1682613921871,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "s-pCBZeeXuAR"
   },
   "outputs": [],
   "source": [
    "model_additive.load_state_dict(torch.load(save_path))\n",
    "\n",
    "model_additive.eval()\n",
    "with torch.no_grad():\n",
    "    output, attentions = model_additive(src_text, text_length, trg_text, 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 266,
     "status": "aborted",
     "timestamp": 1682613921879,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "9vMhJ3pJXvs_"
   },
   "outputs": [],
   "source": [
    "output.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 268,
     "status": "aborted",
     "timestamp": 1682613921882,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "Xw-2Zvf-Xyyq"
   },
   "outputs": [],
   "source": [
    "output = output.squeeze(1)\n",
    "output.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 271,
     "status": "aborted",
     "timestamp": 1682613921885,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "DoAkJ9CSXz4W"
   },
   "outputs": [],
   "source": [
    "output_max = output.argmax(1) #returns max indices\n",
    "output_max"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 271,
     "status": "aborted",
     "timestamp": 1682613921885,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "mqQtW2vvX1Nu"
   },
   "outputs": [],
   "source": [
    "for token in output_max:\n",
    "    print(mapping[token.item()])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "zbdDlSnSX7vb"
   },
   "source": [
    "# 8. Attention"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 271,
     "status": "aborted",
     "timestamp": 1682613921887,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "vqihvOMXX4_R"
   },
   "outputs": [],
   "source": [
    "attentions.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 272,
     "status": "aborted",
     "timestamp": 1682613921888,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "VTnAPHm7X-CS"
   },
   "outputs": [],
   "source": [
    "src_tokens = ['<sos>'] + token_transform[SRC_LANGUAGE].tokenize(dataset[0][1]) + ['<eos>']\n",
    "src_tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 273,
     "status": "aborted",
     "timestamp": 1682613921889,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "iSNiR8LFX_Fk"
   },
   "outputs": [],
   "source": [
    "trg_tokens = ['<sos>'] + [mapping[token.item()] for token in output_max]\n",
    "trg_tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 274,
     "status": "aborted",
     "timestamp": 1682613921890,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "jDiRXSETYANN"
   },
   "outputs": [],
   "source": [
    "import matplotlib.ticker as ticker\n",
    "\n",
    "def display_attention(sentence, translation, attention):\n",
    "    \n",
    "    fig = plt.figure(figsize=(10,10))\n",
    "    ax = fig.add_subplot(111)\n",
    "    \n",
    "    attention = attention.squeeze(1).cpu().detach().numpy()\n",
    "    \n",
    "    cax = ax.matshow(attention, cmap='bone')\n",
    "   \n",
    "    ax.tick_params(labelsize=10)\n",
    "    \n",
    "    y_ticks =  [''] + translation\n",
    "    x_ticks =  [''] + sentence \n",
    "     \n",
    "    ax.set_xticklabels(x_ticks, rotation=45)\n",
    "    ax.set_yticklabels(y_ticks)\n",
    "\n",
    "    ax.xaxis.set_major_locator(ticker.MultipleLocator(1))\n",
    "    ax.yaxis.set_major_locator(ticker.MultipleLocator(1))\n",
    "\n",
    "    plt.show()\n",
    "    plt.close()\n",
    "display_attention(src_tokens, trg_tokens, attentions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 276,
     "status": "aborted",
     "timestamp": 1682613921892,
     "user": {
      "displayName": "Tan Santativongchai",
      "userId": "04436938193154869450"
     },
     "user_tz": -420
    },
    "id": "6_kbv5WwYB-a"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "TPU",
  "colab": {
   "authorship_tag": "ABX9TyNaarKSwpF0dhMcSiiXTu1Q",
   "machine_shape": "hm",
   "mount_file_id": "1_aPXBZVigw6O9Zm3VUwswhtb0BVBELq2",
   "provenance": []
  },
  "gpuClass": "standard",
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}