Natural_Language_Processing/Character-level_recurrent_sequence-to-sequence_model/Character_level_Machine_translator_with_seq2seq_KJS_3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from tqdm.auto import tqdm\n",
    "\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'cpu'"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# device = \"cuda:0\" if torch.cuda.is_available() else \"cpu\"\n",
    "# I get \"RuntimeError: CUDA error: device-side assert triggered\" with cuda. So, I run the code on CPU\n",
    "\n",
    "device = 'cpu'\n",
    "device"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# download the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !!curl -O http://www.manythings.org/anki/fra-eng.zip\n",
    "# !!unzip fra-eng.zip"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 16\n",
    "n_epochs = 20\n",
    "latent_dim = 256 # Latent dimensionality of the encoding space.\n",
    "num_samples = 10000 # Number of samples to train on.\n",
    "\n",
    "# Path to the data txt file on disk.\n",
    "data_path = r'fra.txt'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# prepare dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "# vectorize data\n",
    "\n",
    "input_texts = []\n",
    "target_texts = []\n",
    "\n",
    "input_characters = set()\n",
    "target_characters = set()\n",
    "\n",
    "with open(data_path, 'r') as f:\n",
    "    lines = f.read().split('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of samples: 10000\n",
      "Number of unique input tokens: 71\n",
      "Number of unique output tokens: 93\n",
      "Max sequence length for inputs: 15\n",
      "Max sequence length for outputs: 59\n"
     ]
    }
   ],
   "source": [
    "# read -> eng / french split\n",
    "# input_texts = eng, target_texts = french\n",
    "# lines[: min(num_samples, len(lines)-1)] : num_samples를 lines수보다 높게 잡았을 경우, 에러 방지용 코드\n",
    "# len(lines)-1 : lines가 Read 과정에서 가장 마지막 줄이 공백으로 포함되어 나옴. 따라서, read_data[-2]까지만 사용해야함\n",
    "for i, line in enumerate(lines[: min(num_samples, len(lines)-1)]):\n",
    "    ### line : \"I paid in cash. \tJ'ai payé en espèce.\tCC-BY 2.0 (France) Attribution: tatoeba.org #2877272 (CM) & #1158250 (Wittydev)\"\n",
    "    \n",
    "    try : \n",
    "        input_text, target_text, _ = line.split(sep='\\t')\n",
    "    except:\n",
    "        print(i)\n",
    "#         break\n",
    "    ### input_text : \"I paid in cash.\"\n",
    "    ### target_text : \"J'ai payé en espèce.\"\n",
    "\n",
    "    \n",
    "    # We use \"tab\" as the \"start sequence\" character (\"\\t\" == \"BOS\")\n",
    "    # for the targets, and \"\\n\" as \"end sequence\" character. (\"\\n\" == \"EOS\")\n",
    "    target_text = '\\t' + target_text + '\\n'\n",
    "    input_texts.append(input_text)\n",
    "    target_texts.append(target_text)\n",
    "    ### input_texts : ['I paid in cash.', '', ..]\n",
    "    ### target_texts : [\"\\tJ'ai payé en espèce.\\n\", '\\t+t+\\n', ..]\n",
    "\n",
    "    # word(or sentence) -> characterize!\n",
    "\n",
    "    # put the char into the set type (중복 방지)\n",
    "    for char in input_text:\n",
    "        if char not in input_characters:\n",
    "            input_characters.add(char)\n",
    "            \n",
    "    for char in target_text:\n",
    "        if char not in target_characters:\n",
    "            target_characters.add(char)\n",
    "    ### input_characters = {'c', 'a', 's', 'n', 'p', '.', 'i', 'I', 'd', 'h', '-', '-', ..}\n",
    "    ### target_characters = {'é', \"'\", 'a', 'J', 's', 'c', 'y', 'n', 'e', 'p', '\\t', '.', 'è', '\\n', 'i', ' ', ..}\n",
    "    \n",
    "                \n",
    "# set type -> list type (sorted 유지)\n",
    "# set('d','c','b','a') -> {'a','b','c','d'}\n",
    "# list(set('d','c','b','a')) -> ['a', 'h', 'd', 'b', 'c']   *정렬 깨짐\n",
    "input_characters = sorted(list(input_characters))  # input_characters == encoder token\n",
    "target_characters = sorted(list(target_characters)) # target_charcters == decoder token == '\\t'+target_text+'\\n'\n",
    "### input_characters = [' ', '!', '\"', '$', '%', '&', \"'\", ',', '-', '.', '0', '1', '2', '3', '5', '7', '8', '9', ':', '?', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'Y', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 'é']\n",
    "### target_characters = ['\\t', '\\n', ' ', '!', '$', '%', '&', \"'\", '(', ')', ',', '-', '.', '0', '1', '2', '3', '5', '8', '9', ':', '?', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'Y', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '\\xa0', '«', '»', 'À', 'Ç', 'É', 'Ê', 'à', 'â', 'ç', 'è', 'é', 'ê', 'î', 'ï', 'ô', 'ù', 'û', 'œ', '\\u2009', '’', '\\u202f']\n",
    "\n",
    "num_encoder_tokens = len(input_characters)\n",
    "num_decoder_tokens = len(target_characters)\n",
    "\n",
    "max_encoder_seq_length = max([len(txt) for txt in input_texts])\n",
    "max_decoder_seq_length = max([len(txt) for txt in target_texts])\n",
    "\n",
    "\n",
    "# ## check for max length target sentence == 57?\n",
    "# max_val = 0\n",
    "# max_idx = 0\n",
    "# max_txt = str()\n",
    "# for i, txt in enumerate(target_texts):\n",
    "#     if len(txt) >= max_val:\n",
    "#         max_txt = txt\n",
    "#         max_val = len(txt)\n",
    "#         max_idx = i+1\n",
    "        \n",
    "# print(f\"max length of target sentence : {max_val}, index of line : {max_idx}\",end='\\n \\n')\n",
    "# print(max_txt)\n",
    "\n",
    "print(\"Number of samples:\", len(input_texts))\n",
    "print(\"Number of unique input tokens:\", num_encoder_tokens)\n",
    "print(\"Number of unique output tokens:\", num_decoder_tokens)\n",
    "print(\"Max sequence length for inputs:\", max_encoder_seq_length)\n",
    "print(\"Max sequence length for outputs:\", max_decoder_seq_length)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_token_index = dict([(char, i) for i, char in enumerate(input_characters)])\n",
    "target_token_index = dict([(char, i) for i, char in enumerate(target_characters)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder_input_data = np.zeros(\n",
    "    (len(input_texts), max_encoder_seq_length, num_encoder_tokens), dtype=\"float32\"\n",
    ")\n",
    "decoder_input_data = np.zeros(\n",
    "    (len(input_texts), max_decoder_seq_length, num_decoder_tokens), dtype=\"float32\"\n",
    ")\n",
    "decoder_target_data = np.zeros(\n",
    "    (len(input_texts), max_decoder_seq_length, num_decoder_tokens), dtype=\"float32\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i, (input_text, target_text) in enumerate(zip(input_texts, target_texts)):\n",
    "    \n",
    "    # one-hot vectorization\n",
    "    for t, char in enumerate(input_text):\n",
    "        encoder_input_data[i, t, input_token_index[char]] = 1.0\n",
    "    encoder_input_data[i, t + 1 :, input_token_index[\" \"]] = 1.0\n",
    "    \n",
    "    # one-hot vectorization\n",
    "    for t, char in enumerate(target_text):\n",
    "        # decoder_target_data is ahead of decoder_input_data by one timestep\n",
    "        decoder_input_data[i, t, target_token_index[char]] = 1.0\n",
    "        if t > 0:\n",
    "            # decoder_target_data will be ahead by one timestep\n",
    "            # and will not include the start character. (because t > 0)\n",
    "            decoder_target_data[i, t - 1, target_token_index[char]] = 1.0\n",
    "        \n",
    "    # padding (space char) -> \n",
    "    decoder_input_data[i, t + 1:, target_token_index[\" \"]] = 1.0\n",
    "    # padding (space char) -> [0][0] 에서 2 index가 1로 채워져있다면, padding 된 것.\n",
    "    decoder_target_data[i, t:, target_token_index[\" \"]] = 1.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "|encoder input data| : (10000, 15, 71)\n",
      "|decoder input data| : (10000, 59, 93)\n",
      "|decoder target data| : (10000, 59, 93)\n"
     ]
    }
   ],
   "source": [
    "print(f\"|encoder input data| : {encoder_input_data.shape}\")\n",
    "print(f\"|decoder input data| : {decoder_input_data.shape}\")\n",
    "print(f\"|decoder target data| : {decoder_target_data.shape}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "# To Tensor\n",
    "encoder_input_data = torch.FloatTensor(encoder_input_data)\n",
    "decoder_input_data = torch.FloatTensor(decoder_input_data)\n",
    "decoder_target_data = torch.FloatTensor(decoder_target_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# build the model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## sequence to sequence 구조도"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![title](imgs/seq2seq.jpg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Encoder 구조도"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![title](imgs/seq2seq_encoder.jpg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Encoder flow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![title](imgs/enc_math.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 483,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Encoder(nn.Module):\n",
    "    \n",
    "    def __init__(self, word_vec_size, hidden_size, num_layers=3, dropout_p=.2, bidirectional=True):\n",
    "        \n",
    "        super().__init__()\n",
    "        \n",
    "        self.word_vec_size = word_vec_size\n",
    "        self.hidden_size = int(hidden_size/2)  # to align |hidden size| with |hidden size of decoder(uni-di)|\n",
    "        self.num_layers = num_layers\n",
    "        self.bidirectional = bidirectional\n",
    "        self.dropout_p = dropout_p\n",
    "        \n",
    "        self.lstm = nn.LSTM(\n",
    "            input_size = self.word_vec_size,\n",
    "            hidden_size = self.hidden_size,\n",
    "            num_layers = self.num_layers,\n",
    "            dropout = self.dropout_p,\n",
    "            batch_first = True,\n",
    "            bidirectional = self.bidirectional\n",
    "        )\n",
    "        \n",
    "    def merge_encoder_hiddens(self, out_enc_hidden):\n",
    "        \n",
    "        out_enc_hidden, out_enc_cell = out_enc_hidden[0], out_enc_hidden[1]\n",
    "        \n",
    "        batch_size = out_enc_hidden.size(1) # should be 10000\n",
    "        hidden_size = self.hidden_size # should be 256 (not 128)\n",
    "        \n",
    "        out_enc_hidden = out_enc_hidden.transpose(0,1).contiguous().view(batch_size, -1, \n",
    "                                                                         self.hidden_size*2).transpose(0,1).contiguous()\n",
    "        \n",
    "        out_enc_cell = out_enc_cell.transpose(0,1).contiguous().view(batch_size, -1, \n",
    "                                                                     self.hidden_size*2).transpose(0,1).contiguous()\n",
    "        \n",
    "        return out_enc_hidden, out_enc_cell\n",
    "\n",
    "    def forward(self, x):\n",
    "                \n",
    "        # |x| = (10,000(bs), 15(t-step), 71(|vocab|))\n",
    "        output_enc, out_enc_hidden = self.lstm(x)\n",
    "        # |output_enc| = (10,000(bs), 1, 2*hidden_size*(1/2))\n",
    "        # |output_hidden[0]| = (num_layers * 2(bi-direc), batch_size, hidden_size/2)\n",
    "        \n",
    "        return output_enc, out_enc_hidden"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 484,
   "metadata": {},
   "outputs": [],
   "source": [
    "# encoder = Encoder(encoder_input_data.shape[-1], latent_dim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 485,
   "metadata": {},
   "outputs": [],
   "source": [
    "# output_enc, out_enc_hidden = encoder(encoder_input_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 486,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# print(output_enc.shape)\n",
    "# print(out_enc_hidden[0].shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Decoder 구조도"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![title](imgs/seq2seq_decoder.jpg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Decoder flow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "\n",
    "\n",
    "![title](imgs/dec_math.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 676,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Decoder(nn.Module):\n",
    "    \n",
    "    def __init__(self, word_vec_size, hidden_size, num_layers=3, dropout_p=.2, bidirectional=False):\n",
    "        \n",
    "        super().__init__()\n",
    "        \n",
    "        self.lstm = nn.LSTM(\n",
    "            input_size = word_vec_size + hidden_size, # input_feeding\n",
    "            hidden_size = hidden_size,\n",
    "            num_layers = num_layers,\n",
    "            dropout = dropout_p,\n",
    "            batch_first=True,\n",
    "            bidirectional=False\n",
    "        )\n",
    "        \n",
    "    def forward(self, emb_t, h_t_1_tilde, h_t_1):\n",
    "\n",
    "        '''\n",
    "\n",
    "        Decoder forward는 input_feeding(지난 timestep의 h_tilde값)\n",
    "\n",
    "        1) emb_t : embedding된 decoder input (bs, 1, word_vec_size)\n",
    "\n",
    "        2) h_t_1_tilde : h(t-1) tilde = input_feeding으로 인해 전달될 지난 timestep의 h_tilde값 \n",
    "           (h_tilde : attention에서 얻은 context vector와의 concat[]*W_concat 후 결과)\n",
    "           (bs, 1, hidden_size)\n",
    "\n",
    "        3) h_t_1 : h(t-1) = 이전 t-step의 hidden state값 = (h_t_1, c_t_1) = tuple(이전 t-step의 hidden, cell state)\n",
    "        (#layers, bs, hidden_size)\n",
    "\n",
    "        '''\n",
    "\n",
    "        batch_size = emb_t.shape[0]\n",
    "        hidden_size = h_t_1[0].shape[-1]\n",
    "\n",
    "        ##########==================== for Input feeding ====================##########\n",
    "\n",
    "        # for the first step, we should initialize the output of previous step (input_feeding)\n",
    "        if h_t_1_tilde is None:\n",
    "            \n",
    "            # if this is the first time-step\n",
    "            # emb_t.new : emb_t와 같은 device에서 연산되는 동일한 shape의 새 tensor 생성\n",
    "            h_t_1_tilde = emb_t.new(batch_size, 1, hidden_size).zero_()\n",
    "        \n",
    "        x = torch.cat([emb_t, h_t_1_tilde], dim=-1)\n",
    "\n",
    "        ##########==================== for Input feeding ====================##########\n",
    "\n",
    "        y, h = self.lstm(x, h_t_1)\n",
    "        # |y| = (bs, 1, hs) = |ouput_dec| = (10,000(bs), 1(t-step), hidden_size(uni-direc))\n",
    "        # |h[0]| = |hidden_decoder| = (# layers, bs, hs)\n",
    "\n",
    "        return y, h\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Encoder / Decoder 비교"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```\n",
    "1. Encoder\n",
    "    1) Input\n",
    "        - embedded input source sentence (all t-step) \n",
    "        \n",
    "    2) Return\n",
    "        - last hidden state of encoder\n",
    "    \n",
    "    3) 순차적 진행 여부\n",
    "        - 한 번에 모든 input이 들어감\n",
    "    \n",
    "    \n",
    "2. Decoder\n",
    "    1) Input\n",
    "        - embedded input target sentence (1 t-step) : teacher forcing\n",
    "        - output of prev t-step : input feeding\n",
    "        - hidden state of prev t-step\n",
    "        \n",
    "    2) Return\n",
    "        - output of decoder for 1 t-step\n",
    "        - hidden state\n",
    "        \n",
    "    3) 순차적 진행 여부\n",
    "        - 한 번에 한 t-step 씩 진행됨\n",
    "\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sequence to sequence의 Decoder 특징\n",
    "\n",
    "```\n",
    "\n",
    "1. Auto-regressive task. 즉, Bi-LSTM 불가함 (Uni-LSTM 사용)\n",
    "\n",
    "2. Teacher forcing, Input feeding\n",
    "\n",
    "3. BOS token으로 시작. 따라서, teacher-forcing으로 인한 y1 input이 2 time-step에 input됨 (한 t-step씩 밀려 들어감)\n",
    "\n",
    "```\n",
    "\n",
    "\n",
    "\n",
    "- Auto-regressive task and Teacher-forcing training\n",
    "    - 참고 : https://kh-kim.gitbook.io/natural-language-processing-with-pytorch/00-cover-9/05-teacher-forcing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generator 구조도"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![title](imgs/seq2seq_decoder.jpg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 677,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Generator(nn.Module):\n",
    "    \n",
    "    def __init__(self, hidden_size, output_size):\n",
    "        super().__init__()\n",
    "        \n",
    "        # Linear\n",
    "        self.output = nn.Linear(hidden_size, output_size)\n",
    "        # Log softmax\n",
    "        self.softmax = nn.LogSoftmax(dim=-1)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        # |x| = |output of decoder| = (bs, 1, hs)\n",
    "        z = self.output(x)\n",
    "        # |y| = (bs, 1, output_size)\n",
    "        y = self.softmax(z)\n",
    "        # |y| = (bs, 1, output_size)\n",
    "        \n",
    "        return y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 678,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "|encoder input data| : torch.Size([10000, 15, 71])\n",
      "|decoder input data| : torch.Size([10000, 59, 93])\n",
      "|decoder target data| : torch.Size([10000, 59, 93])\n"
     ]
    }
   ],
   "source": [
    "print(f\"|encoder input data| : {encoder_input_data.shape}\")\n",
    "print(f\"|decoder input data| : {decoder_input_data.shape}\")\n",
    "print(f\"|decoder target data| : {decoder_target_data.shape}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 679,
   "metadata": {},
   "outputs": [],
   "source": [
    "# n_epochs = 10\n",
    "# batch_size = 16\n",
    "use_teacher_forcing = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 680,
   "metadata": {},
   "outputs": [],
   "source": [
    "enc = Encoder(encoder_input_data.shape[-1], latent_dim).to(device)\n",
    "dec = Decoder(decoder_input_data.shape[-1], latent_dim).to(device)\n",
    "gen = Generator(latent_dim, decoder_target_data.shape[-1]).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 681,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Encoder(\n",
      "  (lstm): LSTM(71, 128, num_layers=3, batch_first=True, dropout=0.2, bidirectional=True)\n",
      ")\n",
      "Decoder(\n",
      "  (lstm): LSTM(349, 256, num_layers=3, batch_first=True, dropout=0.2)\n",
      ")\n",
      "Generator(\n",
      "  (output): Linear(in_features=256, out_features=93, bias=True)\n",
      "  (softmax): LogSoftmax(dim=-1)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "print(enc)\n",
    "print(dec)\n",
    "print(gen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 578,
   "metadata": {},
   "outputs": [],
   "source": [
    "enc_optim = optim.RMSprop(enc.parameters())\n",
    "dec_optim = optim.RMSprop(dec.parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 579,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_f = nn.NLLLoss()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## batch 구성"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### train, valid, test split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 580,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([6000, 15, 71]) torch.Size([6000, 59, 93])\n",
      "torch.Size([2000, 15, 71]) torch.Size([2000, 59, 93])\n",
      "torch.Size([2000, 15, 71]) torch.Size([2000, 59, 93])\n"
     ]
    }
   ],
   "source": [
    "# split train, valid, test dataset\n",
    "ratio = [0.6, 0.2, 0.2]\n",
    "train_cnt = int(encoder_input_data.shape[0] * ratio[0])\n",
    "valid_cnt = int(encoder_input_data.shape[0] * ratio[1])\n",
    "test_cnt = int(encoder_input_data.shape[0] * ratio[2])\n",
    "\n",
    "cnts = [train_cnt, valid_cnt, test_cnt]\n",
    "\n",
    "# index_permu = torch.randperm(encoder_input_data.shape[0])\n",
    "\n",
    "enc_inputs = encoder_input_data.split(cnts, dim=0)\n",
    "dec_inputs = decoder_input_data.split(cnts, dim=0)\n",
    "dec_targets = decoder_target_data.split(cnts, dim=0)\n",
    "# enc_inputs[0] : trian set, enc_inputs[1] : dev set, enc_inputs[2] : test set\n",
    "\n",
    "for x_i, y_i in zip(enc_inputs, dec_inputs):\n",
    "    print(x_i.shape, y_i.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### batch split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "375 torch.Size([16, 15, 71])\n",
      "375 torch.Size([16, 59, 93])\n",
      "375 torch.Size([16, 59, 93])\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(n_epochs):\n",
    "    \n",
    "    enc_batchs = enc_inputs[0].split(batch_size, dim=0)\n",
    "    dec_batchs = dec_inputs[0].split(batch_size, dim=0)\n",
    "    tgt_batchs = dec_targets[0].split(batch_size, dim=0)\n",
    "    \n",
    "    print(len(enc_batchs), enc_batchs[0].shape)\n",
    "    print(len(dec_batchs), dec_batchs[0].shape)\n",
    "    print(len(tgt_batchs), tgt_batchs[0].shape)\n",
    "    break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0481d76e54294736bf7825efe706848b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/20 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch : 0, train step start\n",
      "epoch : 0, validation step start\n",
      "epoch : 0 | train loss : 267.727 | dev loss : 15968.322\n",
      "epoch : 1, train step start\n",
      "epoch : 1, validation step start\n",
      "epoch : 1 | train loss : 120.784 | dev loss : 15559.784\n",
      "epoch : 2, train step start\n",
      "epoch : 2, validation step start\n",
      "epoch : 2 | train loss : 118.132 | dev loss : 15542.146\n",
      "epoch : 3, train step start\n",
      "epoch : 3, validation step start\n",
      "epoch : 3 | train loss : 117.547 | dev loss : 15992.714\n",
      "epoch : 4, train step start\n",
      "epoch : 4, validation step start\n",
      "epoch : 4 | train loss : 122.122 | dev loss : 15315.339\n",
      "epoch : 5, train step start\n",
      "epoch : 5, validation step start\n",
      "epoch : 5 | train loss : 116.264 | dev loss : 15231.855\n",
      "epoch : 6, train step start\n"
     ]
    }
   ],
   "source": [
    "train_epoch_loss, dev_epoch_loss = [], []\n",
    "\n",
    "for epoch in tqdm(range(n_epochs)):\n",
    "    \n",
    "    loss = 0\n",
    "    enc_optim.zero_grad()\n",
    "    dec_optim.zero_grad()\n",
    "    \n",
    "    # batch splsit for train set\n",
    "    enc_batchs = enc_inputs[0].to(device).split(batch_size, dim=0)\n",
    "    dec_batchs = dec_inputs[0].to(device).split(batch_size, dim=0)\n",
    "    tgt_batchs = dec_targets[0].to(device).split(batch_size, dim=0)\n",
    "    # |enc_batchs| = (32, 15, 71)\n",
    "    # |dec_batchs| = (32, 59, 93)\n",
    "    # |tgt_batchs| = (32, 59, 93)\n",
    "    \n",
    "    # batch split for dev set\n",
    "    enc_batchs_dev = enc_inputs[1].to(device).split(batch_size, dim=0)\n",
    "    dec_batchs_dev = dec_inputs[1].to(device).split(batch_size, dim=0)\n",
    "    tgt_batchs_dev = dec_targets[1].to(device).split(batch_size, dim=0)\n",
    "    \n",
    "    \n",
    "    # out of memory?\n",
    "#     torch.cuda.empty_cache()\n",
    "        \n",
    "\n",
    "    print(f\"epoch : {epoch}, train step start\")\n",
    "    \n",
    "    if use_teacher_forcing:\n",
    "            \n",
    "        # batch\n",
    "        for enc_batch, dec_batch, tgt_batch in zip(enc_batchs, dec_batchs, tgt_batchs):\n",
    "            \n",
    "            train_loss_batch = 0\n",
    "            \n",
    "            # encoder forward\n",
    "            output_enc, init_hidden_dec = enc(enc_batch)\n",
    "\n",
    "            # encoder dimension change\n",
    "            init_hidden_dec = enc.merge_encoder_hiddens(init_hidden_dec)\n",
    "            \n",
    "            h_t_1_tilde = None\n",
    "            h_t_1 = init_hidden_dec\n",
    "            \n",
    "            # time step\n",
    "            for t_step in range(dec_batchs[0].shape[1]):\n",
    "                \n",
    "                # decoder forward\n",
    "                h_t_1_tilde, h_t_1 = dec(tgt_batch[:, t_step, :].unsqueeze(1), \n",
    "                                         h_t_1_tilde,\n",
    "                                         h_t_1)\n",
    "                # generator\n",
    "                pred = gen(h_t_1_tilde)\n",
    "                \n",
    "                # loss and teacher-forcing\n",
    "                # with this, I get runtime error message like this\n",
    "#                 RuntimeError: 1D target tensor expected, multi-target not supported\n",
    "#                 loss += loss_f(pred.squeeze(1), tgt_batch[:, t_step, :].long())\n",
    "\n",
    "                # loss and teacher-forcing\n",
    "                # It works!\n",
    "                loss += loss_f(pred.squeeze(1), torch.max(tgt_batch[:, t_step, :], 1)[1])\n",
    "            \n",
    "            train_loss_batch += float(loss)\n",
    "    \n",
    "        loss.backward()\n",
    "        \n",
    "        enc_optim.step()\n",
    "        dec_optim.step()\n",
    "        \n",
    "        train_epoch_loss.append(float(loss)/len(enc_batchs))\n",
    "        \n",
    "        \n",
    "        print(f\"epoch : {epoch}, validation step start\")\n",
    "\n",
    "        loss_dev = 0\n",
    "\n",
    "        with torch.no_grad():\n",
    "            for enc_batch, dec_batch, tgt_batch in zip(enc_batchs_dev, dec_batchs_dev, tgt_batchs_dev):\n",
    "\n",
    "                # encoder forward\n",
    "                output_enc_dev, init_hidden_dec_dev = enc(enc_batch)\n",
    "                # encoder dimension change\n",
    "                init_hidden_dec_dev = enc.merge_encoder_hiddens(init_hidden_dec_dev)\n",
    "                                     \n",
    "                h_t_1_tilde = None\n",
    "                h_t_1 = init_hidden_dec_dev\n",
    "                \n",
    "                # time step\n",
    "                for t_step in range(dec_batchs[1].shape[1]):\n",
    "\n",
    "                    # decoder forward\n",
    "                    h_t_1_tilde, h_t_1 = dec(tgt_batch[:, t_step, :].unsqueeze(1), \n",
    "                                             h_t_1_tilde,\n",
    "                                             h_t_1)\n",
    "                    # generator\n",
    "                    pred = gen(h_t_1_tilde)\n",
    "\n",
    "                    # loss and teacher-forcing\n",
    "                    loss_dev += loss_f(pred.squeeze(1), torch.max(tgt_batch[:, t_step, :], 1)[1])\n",
    "                    \n",
    "                    \n",
    "            dev_epoch_loss.append(float(loss_dev))\n",
    "\n",
    "    print(\"epoch : {} | train loss : {:.3f} | dev loss : {:.3f}\".format(epoch, \n",
    "                                                                        train_epoch_loss[-1],\n",
    "                                                                        dev_epoch_loss[-1]))\n",
    "    \n",
    "\n",
    "# reference : https://justkode.kr/deep-learning/pytorch-save\n",
    "# save model for inference\n",
    "PATH = 'result/'\n",
    "\n",
    "# 모델 자체를 저장\n",
    "torch.save(enc, PATH + 'enc.pt')  # 전체 모델 저장\n",
    "torch.save(dec, PATH + 'dec.pt')  # 전체 모델 저장\n",
    "torch.save(gen, PATH + 'gen.pt')  # 전체 모델 저장\n",
    "\n",
    "# state_dict로 모델과 optimizer 저장\n",
    "torch.save(enc.state_dict(), PATH + 'enc_model_state_dict.pt')  # 모델 객체의 state_dict 저장\n",
    "torch.save(dec.state_dict(), PATH + 'dec_model_state_dict.pt')  # 모델 객체의 state_dict 저장\n",
    "torch.save(gen.state_dict(), PATH + 'gen_model_state_dict.pt')  # 모델 객체의 state_dict 저장\n",
    "\n",
    "torch.save({\n",
    "    'enc_model': enc.state_dict(),\n",
    "    'dec_model': dec.state_dict(),\n",
    "    'gen_model': gen.state_dict(),\n",
    "    'enc_optimizer': enc_optim.state_dict(),\n",
    "    'dec_optimizer': dec_optim.state_dict()}, \n",
    "    PATH + 'all.tar')  # 여러 가지 값 저장, 학습 중 진행 상황 저장을 위해 epoch, loss 값 등 일반 scalar값 저장 가능"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## model_load"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 531,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/home/jeongseobkim/바탕화면/pytorch_reboost/Machine_translator'"
      ]
     },
     "execution_count": 531,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 720,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model load\n",
    "\n",
    "PATH = 'result/'\n",
    "\n",
    "# 방법 1 : 전체 모델을 통쨰로 불러오기\n",
    "enc = torch.load(PATH + 'enc.pt' )\n",
    "dec = torch.load(PATH + 'dec.pt' )\n",
    "gen = torch.load(PATH + 'gen.pt' )\n",
    "\n",
    "# 방법 2 : state_dict를 불러온 후, 모델에 저장\n",
    "enc.load_state_dict(torch.load(PATH + 'enc_model_state_dict.pt'))  # 모델 객체의 state_dict 저장\n",
    "dec.load_state_dict(torch.load(PATH + 'dec_model_state_dict.pt'))  # 모델 객체의 state_dict 저장\n",
    "gen.load_state_dict(torch.load(PATH + 'gen_model_state_dict.pt'))  # 모델 객체의 state_dict 저장\n",
    "\n",
    "# optimizer load\n",
    "checkpoint = torch.load(PATH + 'all.tar')   # dict 불러오기\n",
    "enc_optim.load_state_dict(checkpoint['enc_optimizer'])\n",
    "dec_optim.load_state_dict(checkpoint['dec_optimizer'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 721,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Encoder(\n",
       "  (lstm): LSTM(71, 128, num_layers=3, batch_first=True, dropout=0.2, bidirectional=True)\n",
       ")"
      ]
     },
     "execution_count": 721,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "enc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 722,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Decoder(\n",
       "  (lstm): LSTM(349, 256, num_layers=3, batch_first=True, dropout=0.2)\n",
       ")"
      ]
     },
     "execution_count": 722,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 723,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Generator(\n",
       "  (output): Linear(in_features=256, out_features=93, bias=True)\n",
       "  (softmax): LogSoftmax(dim=-1)\n",
       ")"
      ]
     },
     "execution_count": 723,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 724,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RMSprop (\n",
       "Parameter Group 0\n",
       "    alpha: 0.99\n",
       "    centered: False\n",
       "    eps: 1e-08\n",
       "    lr: 0.01\n",
       "    momentum: 0\n",
       "    weight_decay: 0\n",
       ")"
      ]
     },
     "execution_count": 724,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "enc_optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 725,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RMSprop (\n",
       "Parameter Group 0\n",
       "    alpha: 0.99\n",
       "    centered: False\n",
       "    eps: 1e-08\n",
       "    lr: 0.01\n",
       "    momentum: 0\n",
       "    weight_decay: 0\n",
       ")"
      ]
     },
     "execution_count": 725,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dec_optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 726,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "enc_model\n",
      "dec_model\n",
      "gen_model\n",
      "enc_optimizer\n",
      "dec_optimizer\n"
     ]
    }
   ],
   "source": [
    "for key in checkpoint.keys():\n",
    "    print(key)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 727,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'lr': 0.01,\n",
       "  'momentum': 0,\n",
       "  'alpha': 0.99,\n",
       "  'eps': 1e-08,\n",
       "  'centered': False,\n",
       "  'weight_decay': 0,\n",
       "  'params': [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]}]"
      ]
     },
     "execution_count": 727,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "checkpoint['dec_optimizer']['param_groups']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 728,
   "metadata": {},
   "outputs": [],
   "source": [
    "# checkpoint['dec_optimizer']['state']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## inference"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### recap data prerpocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 729,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "enc test data :  torch.Size([2000, 15, 71])\n",
      "dec test data :  torch.Size([2000, 59, 93])\n",
      "gen test data :  torch.Size([2000, 59, 93])\n"
     ]
    }
   ],
   "source": [
    "# check test inputs and targets\n",
    "print(\"enc test data : \", enc_inputs[2].shape)\n",
    "print(\"dec test data : \", dec_inputs[2].shape)\n",
    "print(\"dec_target test data : \", dec_targets[2].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 730,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{' ': 0, '!': 1, '\"': 2, '$': 3, '%': 4, '&': 5, \"'\": 6, ',': 7, '-': 8, '.': 9, '0': 10, '1': 11, '2': 12, '3': 13, '5': 14, '7': 15, '8': 16, '9': 17, ':': 18, '?': 19, 'A': 20, 'B': 21, 'C': 22, 'D': 23, 'E': 24, 'F': 25, 'G': 26, 'H': 27, 'I': 28, 'J': 29, 'K': 30, 'L': 31, 'M': 32, 'N': 33, 'O': 34, 'P': 35, 'Q': 36, 'R': 37, 'S': 38, 'T': 39, 'U': 40, 'V': 41, 'W': 42, 'Y': 43, 'a': 44, 'b': 45, 'c': 46, 'd': 47, 'e': 48, 'f': 49, 'g': 50, 'h': 51, 'i': 52, 'j': 53, 'k': 54, 'l': 55, 'm': 56, 'n': 57, 'o': 58, 'p': 59, 'q': 60, 'r': 61, 's': 62, 't': 63, 'u': 64, 'v': 65, 'w': 66, 'x': 67, 'y': 68, 'z': 69, 'é': 70}\n"
     ]
    }
   ],
   "source": [
    "print(input_token_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 731,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'\\t': 0, '\\n': 1, ' ': 2, '!': 3, '$': 4, '%': 5, '&': 6, \"'\": 7, '(': 8, ')': 9, ',': 10, '-': 11, '.': 12, '0': 13, '1': 14, '2': 15, '3': 16, '5': 17, '8': 18, '9': 19, ':': 20, '?': 21, 'A': 22, 'B': 23, 'C': 24, 'D': 25, 'E': 26, 'F': 27, 'G': 28, 'H': 29, 'I': 30, 'J': 31, 'K': 32, 'L': 33, 'M': 34, 'N': 35, 'O': 36, 'P': 37, 'Q': 38, 'R': 39, 'S': 40, 'T': 41, 'U': 42, 'V': 43, 'Y': 44, 'a': 45, 'b': 46, 'c': 47, 'd': 48, 'e': 49, 'f': 50, 'g': 51, 'h': 52, 'i': 53, 'j': 54, 'k': 55, 'l': 56, 'm': 57, 'n': 58, 'o': 59, 'p': 60, 'q': 61, 'r': 62, 's': 63, 't': 64, 'u': 65, 'v': 66, 'w': 67, 'x': 68, 'y': 69, 'z': 70, '\\xa0': 71, '«': 72, '»': 73, 'À': 74, 'Ç': 75, 'É': 76, 'Ê': 77, 'à': 78, 'â': 79, 'ç': 80, 'è': 81, 'é': 82, 'ê': 83, 'î': 84, 'ï': 85, 'ô': 86, 'ù': 87, 'û': 88, 'œ': 89, '\\u2009': 90, '’': 91, '\\u202f': 92}\n"
     ]
    }
   ],
   "source": [
    "print(target_token_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 732,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 732,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# \"\\t\" == \"BOS\"\n",
    "target_token_index['\\t']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 733,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 733,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# \"\\n\" == \"EOS\"\n",
    "target_token_index['\\n']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 734,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "inputs sentence sample :  10000\n",
      "targets sentence sample :  10000\n",
      "inputs sample 1 :  I paid.\n",
      "targets sample 1 :  \tJe payai.\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'\\tJe payai.\\n'"
      ]
     },
     "execution_count": 734,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 데이터 전처리 다시 살펴보기\n",
    "### We use \"tab\" as the \"start sequence\" character (\"\\t\" == \"BOS\")\n",
    "### for the targets, and \"\\n\" as \"end sequence\" character. (\"\\n\" == \"EOS\")\n",
    "print(\"inputs sentence sample : \", len(input_texts))\n",
    "print(\"targets sentence sample : \", len(target_texts))\n",
    "print(\"inputs sample 1 : \", input_texts[100])\n",
    "print(\"targets sample 1 : \", target_texts[100])\n",
    "\n",
    "# \\t : BOS, \\n : EOS\n",
    "target_texts[100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 735,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([59, 93])"
      ]
     },
     "execution_count": 735,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "decoder_input_data[100].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 736,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0])"
      ]
     },
     "execution_count": 736,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.nonzero(decoder_input_data[100][0])[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 737,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([1])"
      ]
     },
     "execution_count": 737,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.nonzero(decoder_input_data[100][10])[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 738,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([2])"
      ]
     },
     "execution_count": 738,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.nonzero(decoder_input_data[100][11])[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 739,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([2])"
      ]
     },
     "execution_count": 739,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.nonzero(decoder_input_data[100][58])[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 740,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Go.'"
      ]
     },
     "execution_count": 740,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_texts[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 741,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\tVa !\\n'"
      ]
     },
     "execution_count": 741,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target_texts[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 742,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\tVa !\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(target_texts[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reverse-lookup token index to decode sequences back to something readable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 743,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{' ': 0, '!': 1, '\"': 2, '$': 3, '%': 4, '&': 5, \"'\": 6, ',': 7, '-': 8, '.': 9, '0': 10, '1': 11, '2': 12, '3': 13, '5': 14, '7': 15, '8': 16, '9': 17, ':': 18, '?': 19, 'A': 20, 'B': 21, 'C': 22, 'D': 23, 'E': 24, 'F': 25, 'G': 26, 'H': 27, 'I': 28, 'J': 29, 'K': 30, 'L': 31, 'M': 32, 'N': 33, 'O': 34, 'P': 35, 'Q': 36, 'R': 37, 'S': 38, 'T': 39, 'U': 40, 'V': 41, 'W': 42, 'Y': 43, 'a': 44, 'b': 45, 'c': 46, 'd': 47, 'e': 48, 'f': 49, 'g': 50, 'h': 51, 'i': 52, 'j': 53, 'k': 54, 'l': 55, 'm': 56, 'n': 57, 'o': 58, 'p': 59, 'q': 60, 'r': 61, 's': 62, 't': 63, 'u': 64, 'v': 65, 'w': 66, 'x': 67, 'y': 68, 'z': 69, 'é': 70}\n"
     ]
    }
   ],
   "source": [
    "print(input_token_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 744,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'\\t': 0, '\\n': 1, ' ': 2, '!': 3, '$': 4, '%': 5, '&': 6, \"'\": 7, '(': 8, ')': 9, ',': 10, '-': 11, '.': 12, '0': 13, '1': 14, '2': 15, '3': 16, '5': 17, '8': 18, '9': 19, ':': 20, '?': 21, 'A': 22, 'B': 23, 'C': 24, 'D': 25, 'E': 26, 'F': 27, 'G': 28, 'H': 29, 'I': 30, 'J': 31, 'K': 32, 'L': 33, 'M': 34, 'N': 35, 'O': 36, 'P': 37, 'Q': 38, 'R': 39, 'S': 40, 'T': 41, 'U': 42, 'V': 43, 'Y': 44, 'a': 45, 'b': 46, 'c': 47, 'd': 48, 'e': 49, 'f': 50, 'g': 51, 'h': 52, 'i': 53, 'j': 54, 'k': 55, 'l': 56, 'm': 57, 'n': 58, 'o': 59, 'p': 60, 'q': 61, 'r': 62, 's': 63, 't': 64, 'u': 65, 'v': 66, 'w': 67, 'x': 68, 'y': 69, 'z': 70, '\\xa0': 71, '«': 72, '»': 73, 'À': 74, 'Ç': 75, 'É': 76, 'Ê': 77, 'à': 78, 'â': 79, 'ç': 80, 'è': 81, 'é': 82, 'ê': 83, 'î': 84, 'ï': 85, 'ô': 86, 'ù': 87, 'û': 88, 'œ': 89, '\\u2009': 90, '’': 91, '\\u202f': 92}\n"
     ]
    }
   ],
   "source": [
    "print(target_token_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 745,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 'reverse' means \n",
    "# \"\\t : 0\" \n",
    "# ---->>>>> \n",
    "# \"0 : \\t\" \n",
    "# for recognize the one-hot vectorized token\n",
    "\n",
    "reverse_input_char_index = dict((i, char) for char, i in input_token_index.items())\n",
    "reverse_target_char_index = dict((i, char) for char, i in target_token_index.items())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 746,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'\\t': 0, '\\n': 1, ' ': 2, '!': 3, '$': 4, '%': 5, '&': 6, \"'\": 7, '(': 8, ')': 9, ',': 10, '-': 11, '.': 12, '0': 13, '1': 14, '2': 15, '3': 16, '5': 17, '8': 18, '9': 19, ':': 20, '?': 21, 'A': 22, 'B': 23, 'C': 24, 'D': 25, 'E': 26, 'F': 27, 'G': 28, 'H': 29, 'I': 30, 'J': 31, 'K': 32, 'L': 33, 'M': 34, 'N': 35, 'O': 36, 'P': 37, 'Q': 38, 'R': 39, 'S': 40, 'T': 41, 'U': 42, 'V': 43, 'Y': 44, 'a': 45, 'b': 46, 'c': 47, 'd': 48, 'e': 49, 'f': 50, 'g': 51, 'h': 52, 'i': 53, 'j': 54, 'k': 55, 'l': 56, 'm': 57, 'n': 58, 'o': 59, 'p': 60, 'q': 61, 'r': 62, 's': 63, 't': 64, 'u': 65, 'v': 66, 'w': 67, 'x': 68, 'y': 69, 'z': 70, '\\xa0': 71, '«': 72, '»': 73, 'À': 74, 'Ç': 75, 'É': 76, 'Ê': 77, 'à': 78, 'â': 79, 'ç': 80, 'è': 81, 'é': 82, 'ê': 83, 'î': 84, 'ï': 85, 'ô': 86, 'ù': 87, 'û': 88, 'œ': 89, '\\u2009': 90, '’': 91, '\\u202f': 92}\n"
     ]
    }
   ],
   "source": [
    "print(target_token_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 747,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{0: '\\t', 1: '\\n', 2: ' ', 3: '!', 4: '$', 5: '%', 6: '&', 7: \"'\", 8: '(', 9: ')', 10: ',', 11: '-', 12: '.', 13: '0', 14: '1', 15: '2', 16: '3', 17: '5', 18: '8', 19: '9', 20: ':', 21: '?', 22: 'A', 23: 'B', 24: 'C', 25: 'D', 26: 'E', 27: 'F', 28: 'G', 29: 'H', 30: 'I', 31: 'J', 32: 'K', 33: 'L', 34: 'M', 35: 'N', 36: 'O', 37: 'P', 38: 'Q', 39: 'R', 40: 'S', 41: 'T', 42: 'U', 43: 'V', 44: 'Y', 45: 'a', 46: 'b', 47: 'c', 48: 'd', 49: 'e', 50: 'f', 51: 'g', 52: 'h', 53: 'i', 54: 'j', 55: 'k', 56: 'l', 57: 'm', 58: 'n', 59: 'o', 60: 'p', 61: 'q', 62: 'r', 63: 's', 64: 't', 65: 'u', 66: 'v', 67: 'w', 68: 'x', 69: 'y', 70: 'z', 71: '\\xa0', 72: '«', 73: '»', 74: 'À', 75: 'Ç', 76: 'É', 77: 'Ê', 78: 'à', 79: 'â', 80: 'ç', 81: 'è', 82: 'é', 83: 'ê', 84: 'î', 85: 'ï', 86: 'ô', 87: 'ù', 88: 'û', 89: 'œ', 90: '\\u2009', 91: '’', 92: '\\u202f'}\n"
     ]
    }
   ],
   "source": [
    "print(reverse_target_char_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 748,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'l'"
      ]
     },
     "execution_count": 748,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reverse_target_char_index[56]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### make decode_sequence function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 749,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num_decoder_tokens :  93\n",
      "target_token_index['\\t'] : 0\n",
      "target_token_index['\\n'] : 1\n"
     ]
    }
   ],
   "source": [
    "print(\"num_decoder_tokens : \",num_decoder_tokens)\n",
    "print(\"target_token_index['\\\\t'] :\",target_token_index['\\t'])\n",
    "print(\"target_token_index['\\\\n'] :\",target_token_index['\\n'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 750,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def decode_sequence(input_seq):\n",
    "    \n",
    "#     # Encode the input as state vectors.\n",
    "#     _, init_hidden_dec = enc(input_seq)\n",
    "    \n",
    "#     # encoder dimension change\n",
    "#     init_hidden_dec = enc.merge_encoder_hiddens(init_hidden_dec)\n",
    "    \n",
    "#     # Generate 'empty' target sequence of length 1.\n",
    "#     target_seq = np.zeros((1, 1, num_decoder_tokens)) # num_decoder_tokens = 93\n",
    "    \n",
    "#     # fill the first character of target sequence with the start character('\\t').\n",
    "#     # (0,0,0) = (1 sample, 1 time-step, start!)\n",
    "#     # target_seq[0, 0, 0] = \"start of target sequence!\" = BOS token input\n",
    "#     target_seq[0, 0, target_token_index[\"\\t\"]] = 1.0\n",
    "\n",
    "#     # Sampling loop for a batch of sequences\n",
    "#     # (to simplify, here we assume a batch of size 1).\n",
    "#     stop_condition = False\n",
    "#     decoded_sentence = \"\"\n",
    "    \n",
    "#     h_t_1_tilde = None\n",
    "#     h_t_1 = init_hidden_dec\n",
    "    \n",
    "#     while not stop_condition:\n",
    "        \n",
    "#         # delete\n",
    "# #         output_tokens, h, c = decoder_model.predict([target_seq] + states_value)\n",
    "#         target_seq = torch.FloatTensor(target_seq)\n",
    "\n",
    "#         # get output of 1 step, hidden state\n",
    "#         h_t_1_tilde, h_t_1 = dec(target_seq, h_t_1_tilde, h_t_1)\n",
    "#         # |h_t_1_tilde| = (bs=1, 1, hs)\n",
    "        \n",
    "#         # Sample a token\n",
    "#         # delete\n",
    "# #         sampled_token_index = np.argmax(output_tokens[0, -1, :])\n",
    "        \n",
    "# #         print(f\"h_t_1_tilde shape : {h_t_1_tilde.shape}\")\n",
    "# #         print(f\"torch.argmax(h_t_1_tilde[0,-1,:]) : {torch.argmax(h_t_1_tilde[0, -1, :])}\")\n",
    "\n",
    "#         sampled_token_index = torch.argmax(h_t_1_tilde[0, -1, :])\n",
    "#         print(h_t_1_tilde[0, -1, :].shape)\n",
    "#         print(f\"sampled_token_index : {sampled_token_index}\")\n",
    "# #         print(f\"target_seq before : {target_seq}\")\n",
    "#         print(f\"target_seq before shape : {target_seq.shape}\")\n",
    "\n",
    "#         sampled_char = reverse_target_char_index[sampled_token_index.tolist()]\n",
    "#         print(f\"sampled_char : {sampled_char}\")\n",
    "#         decoded_sentence += sampled_char\n",
    "#         print(f\"decoded_sentence : {decoded_sentence}\")\n",
    "\n",
    "#         # Exit condition: either hit max length\n",
    "#         # or find stop character.\n",
    "#         if sampled_char == \"\\n\" or len(decoded_sentence) > max_decoder_seq_length:\n",
    "#             stop_condition = True\n",
    "\n",
    "#         # Update the target sequence (of length 1).\n",
    "#         target_seq = np.zeros((1, 1, num_decoder_tokens))\n",
    "#         target_seq[0, 0, sampled_token_index] = 1.0\n",
    "        \n",
    "# #         print(f\"target_seq after : {target_seq}\")\n",
    "#         print(f\"target_seq after shape : {target_seq.shape}\")\n",
    "\n",
    "#     return decoded_sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 751,
   "metadata": {},
   "outputs": [],
   "source": [
    "# debugging\n",
    "h_t_1_tilde_list = [] \n",
    "\n",
    "def decode_sequence(input_seq):\n",
    "    \n",
    "    # Encode the input as state vectors.\n",
    "    _, init_hidden_dec = enc(input_seq)\n",
    "    \n",
    "    # encoder dimension change\n",
    "    init_hidden_dec = enc.merge_encoder_hiddens(init_hidden_dec)\n",
    "    \n",
    "    # Generate 'empty' target sequence of length 1.\n",
    "    target_seq = np.zeros((1, 1, num_decoder_tokens)) # num_decoder_tokens = 93\n",
    "    \n",
    "    # fill the first character of target sequence with the start character('\\t').\n",
    "    # (0,0,0) = (1 sample, 1 time-step, start!)\n",
    "    # target_seq[0, 0, 0] = \"start of target sequence!\" = BOS token input\n",
    "    target_seq[0, 0, target_token_index[\"\\t\"]] = 1.0\n",
    "\n",
    "    # Sampling loop for a batch of sequences\n",
    "    # (to simplify, here we assume a batch of size 1).\n",
    "    stop_condition = False\n",
    "    decoded_sentence = \"\"\n",
    "    \n",
    "    h_t_1_tilde = None\n",
    "    h_t_1 = init_hidden_dec\n",
    "    \n",
    "    while not stop_condition:\n",
    "\n",
    "        target_seq = torch.FloatTensor(target_seq)\n",
    "\n",
    "        # get output of 1 step, hidden state\n",
    "        h_t_1_tilde, h_t_1 = dec(target_seq, h_t_1_tilde, h_t_1)\n",
    "        \n",
    "        h_t_1_tilde_list.append(h_t_1_tilde)\n",
    "        # |h_t_1_tilde| = (bs=1, 1, hs)\n",
    "        \n",
    "        # Sample a token\n",
    "        sampled_token_index = torch.argmax(gen(h_t_1_tilde[0, -1, :]))\n",
    "\n",
    "        sampled_char = reverse_target_char_index[sampled_token_index.tolist()]\n",
    "        decoded_sentence += sampled_char\n",
    "\n",
    "        # Exit condition: either hit max length\n",
    "        # or find stop character.\n",
    "        if sampled_char == \"\\n\" or len(decoded_sentence) > max_decoder_seq_length:\n",
    "            stop_condition = True\n",
    "\n",
    "        # Update the target sequence (of length 1).\n",
    "        target_seq = np.zeros((1, 1, num_decoder_tokens))\n",
    "        target_seq[0, 0, sampled_token_index] = 1.0\n",
    "\n",
    "    return decoded_sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 752,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-\n",
      "Input sentence: I'm OK.\n",
      "Decoded sentence: J                                                           \n",
      "-\n",
      "Input sentence: Listen.\n",
      "Decoded sentence: J                                                           \n",
      "-\n",
      "Input sentence: No way!\n",
      "Decoded sentence: J                                                           \n",
      "-\n",
      "Input sentence: No way!\n",
      "Decoded sentence: J                                                           \n",
      "-\n",
      "Input sentence: No way!\n",
      "Decoded sentence: J                                                           \n",
      "-\n",
      "Input sentence: No way!\n",
      "Decoded sentence: J                                                           \n"
     ]
    }
   ],
   "source": [
    "for seq_index in range(105,111):\n",
    "    # Take one sequence (part of the training set)\n",
    "    # for trying out decoding.\n",
    "    input_seq = encoder_input_data[seq_index : seq_index + 1]\n",
    "    decoded_sentence = decode_sequence(input_seq)\n",
    "    print(\"-\")\n",
    "    print(\"Input sentence:\", input_texts[seq_index])\n",
    "    print(\"Decoded sentence:\", decoded_sentence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 754,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[ True, False, False, False, False, False, False, False, False, False,\n",
       "          False, False, False, False, False, False, False, False, False, False,\n",
       "          False, False, False, False, False, False, False, False, False, False,\n",
       "          False, False, False, False, False, False, False, False, False, False,\n",
       "          False, False, False, False, False, False, False, False, False, False,\n",
       "          False, False, False, False, False, False,  True, False, False, False,\n",
       "          False, False, False, False, False, False, False, False, False, False,\n",
       "          False, False, False, False, False, False, False, False, False, False,\n",
       "          False, False, False, False, False, False, False, False, False, False,\n",
       "          False, False, False, False, False, False,  True, False, False, False,\n",
       "          False, False, False, False, False, False, False, False, False, False,\n",
       "          False, False, False, False, False, False,  True, False, False, False,\n",
       "          False, False, False, False, False, False, False, False, False, False,\n",
       "          False, False, False, False, False, False, False, False, False, False,\n",
       "          False, False, False, False, False, False, False, False, False,  True,\n",
       "          False, False, False, False, False, False, False, False, False, False,\n",
       "          False, False, False, False, False, False, False, False, False, False,\n",
       "          False, False, False, False, False, False, False, False, False, False,\n",
       "           True, False, False, False, False, False, False, False, False, False,\n",
       "          False, False, False, False, False, False, False, False, False, False,\n",
       "          False, False, False, False, False, False, False, False, False, False,\n",
       "          False, False, False, False, False, False, False, False, False, False,\n",
       "          False, False, False, False, False, False, False, False, False, False,\n",
       "          False, False, False, False, False, False, False, False, False, False,\n",
       "          False, False, False, False, False, False, False, False, False,  True,\n",
       "          False, False, False, False, False, False]]])"
      ]
     },
     "execution_count": 754,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "h_t_1_tilde_list[0] == h_t_1_tilde_list[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 755,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[False,  True,  True, False, False, False, False, False, False, False,\n",
       "           True,  True,  True,  True, False, False, False,  True, False, False,\n",
       "          False, False, False, False,  True, False, False, False, False, False,\n",
       "          False, False, False, False, False,  True, False,  True, False, False,\n",
       "          False, False, False, False, False, False, False, False,  True,  True,\n",
       "          False, False, False, False, False, False,  True, False, False, False,\n",
       "           True, False, False, False,  True, False, False, False, False, False,\n",
       "          False, False,  True, False, False,  True, False, False, False, False,\n",
       "          False, False, False, False, False, False, False, False, False, False,\n",
       "          False,  True, False, False, False,  True,  True, False, False,  True,\n",
       "           True,  True, False, False, False,  True,  True,  True, False, False,\n",
       "          False, False,  True, False, False,  True, False,  True,  True, False,\n",
       "          False, False, False,  True, False, False, False, False, False, False,\n",
       "          False, False, False, False, False, False,  True, False,  True,  True,\n",
       "          False, False, False,  True, False, False, False, False,  True, False,\n",
       "          False, False, False, False, False, False, False, False,  True, False,\n",
       "           True, False, False,  True, False, False, False, False, False, False,\n",
       "          False,  True, False,  True, False, False,  True,  True, False, False,\n",
       "          False,  True, False,  True, False,  True, False, False, False,  True,\n",
       "          False, False,  True, False,  True, False,  True, False, False, False,\n",
       "           True, False, False,  True, False, False, False,  True,  True, False,\n",
       "          False,  True, False, False, False,  True, False, False, False, False,\n",
       "          False, False, False, False, False, False, False, False,  True, False,\n",
       "           True, False, False,  True, False, False, False, False,  True,  True,\n",
       "           True, False, False,  True, False,  True, False, False, False, False,\n",
       "          False, False, False, False, False,  True]]])"
      ]
     },
     "execution_count": 755,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "h_t_1_tilde_list[4] == h_t_1_tilde_list[5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}