5-2_model_imdb_scratch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 85
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 4043,
     "status": "ok",
     "timestamp": 1593193785731,
     "user": {
      "displayName": "seongsu bang",
      "photoUrl": "",
      "userId": "06124410414614761416"
     },
     "user_tz": -540
    },
    "id": "2XWOHqwyv9Kn",
    "outputId": "2624bca6-d525-4c5d-a7b2-4ffb04f4440b"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "'grep'은(는) 내부 또는 외부 명령, 실행할 수 있는 프로그램, 또는\n",
      "배치 파일이 아닙니다.\n"
     ]
    }
   ],
   "source": [
    "! pip list | grep \"torch\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 974,
     "status": "ok",
     "timestamp": 1593193218656,
     "user": {
      "displayName": "seongsu bang",
      "photoUrl": "",
      "userId": "06124410414614761416"
     },
     "user_tz": -540
    },
    "id": "aJGwgEKR_GPg"
   },
   "outputs": [],
   "source": [
    "import re\n",
    "import sys\n",
    "import random\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "from torchtext import data\n",
    "from torchtext import datasets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "M3O6ttcHKrIq"
   },
   "source": [
    "## Reading Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 41518,
     "status": "ok",
     "timestamp": 1593193202383,
     "user": {
      "displayName": "seongsu bang",
      "photoUrl": "",
      "userId": "06124410414614761416"
     },
     "user_tz": -540
    },
    "id": "1oJjy5xBJXAx",
    "outputId": "54ba92fa-db22-4f17-f175-737ce1f1fe7f"
   },
   "outputs": [],
   "source": [
    "\n",
    "# Data Setting\n",
    "TEXT = data.Field(batch_first = True,\n",
    "                  fix_length = 500,\n",
    "                  tokenize=str.split,\n",
    "                  pad_first=True,\n",
    "                  pad_token='[PAD]',\n",
    "                  unk_token='[UNK]')\n",
    "\n",
    "LABEL = data.LabelField(dtype=torch.float)\n",
    "\n",
    "train_data, test_data = datasets.IMDB.splits(text_field = TEXT, \n",
    "                                             label_field = LABEL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 998,
     "status": "ok",
     "timestamp": 1593193204453,
     "user": {
      "displayName": "seongsu bang",
      "photoUrl": "",
      "userId": "06124410414614761416"
     },
     "user_tz": -540
    },
    "id": "34DoliQ6mdob",
    "outputId": "4506a83c-9078-497b-ee63-e0819e53e963"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Data Length : 25000\n",
      "Test Data Length : 25000\n"
     ]
    }
   ],
   "source": [
    "# Data Length\n",
    "print(f'Train Data Length : {len(train_data.examples)}')\n",
    "print(f'Test Data Length : {len(test_data.examples)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 507,
     "status": "ok",
     "timestamp": 1593193204454,
     "user": {
      "displayName": "seongsu bang",
      "photoUrl": "",
      "userId": "06124410414614761416"
     },
     "user_tz": -540
    },
    "id": "3UNJJZE0mH75",
    "outputId": "d759aad6-3367-4ad1-d093-8f47b89d814c"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'text': <torchtext.data.field.Field at 0x27a245fc5c0>,\n",
       " 'label': <torchtext.data.field.LabelField at 0x27a245fc588>}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Data Fields\n",
    "train_data.fields"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 139
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1089,
     "status": "ok",
     "timestamp": 1593193205912,
     "user": {
      "displayName": "seongsu bang",
      "photoUrl": "",
      "userId": "06124410414614761416"
     },
     "user_tz": -540
    },
    "id": "ViUZ13AM_HFQ",
    "outputId": "1eb1bb98-2c37-46f4-b641-c48f65623cda"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---- Data Sample ----\n",
      "Input : \n",
      "Homelessness (or Houselessness as George Carlin stated) has been an issue for years but never a plan to help those on the street that were once considered human who did everything from going to school, work, or vote for the matter. Most people think of the homeless as just a lost cause while worrying about things such as racism, the war on Iraq, pressuring kids to succeed, technology, the elections, inflation, or worrying if they'll be next to end up on the streets.<br /><br />But what if you were given a bet to live on the streets for a month without the luxuries you once had from a home, the entertainment sets, a bathroom, pictures on the wall, a computer, and everything you once treasure to see what it's like to be homeless? That is Goddard Bolt's lesson.<br /><br />Mel Brooks (who directs) who stars as Bolt plays a rich man who has everything in the world until deciding to make a bet with a sissy rival (Jeffery Tambor) to see if he can live in the streets for thirty days without the luxuries; if Bolt succeeds, he can do what he wants with a future project of making more buildings. The bet's on where Bolt is thrown on the street with a bracelet on his leg to monitor his every move where he can't step off the sidewalk. He's given the nickname Pepto by a vagrant after it's written on his forehead where Bolt meets other characters including a woman by the name of Molly (Lesley Ann Warren) an ex-dancer who got divorce before losing her home, and her pals Sailor (Howard Morris) and Fumes (Teddy Wilson) who are already used to the streets. They're survivors. Bolt isn't. He's not used to reaching mutual agreements like he once did when being rich where it's fight or flight, kill or be killed.<br /><br />While the love connection between Molly and Bolt wasn't necessary to plot, I found \"Life Stinks\" to be one of Mel Brooks' observant films where prior to being a comedy, it shows a tender side compared to his slapstick work such as Blazing Saddles, Young Frankenstein, or Spaceballs for the matter, to show what it's like having something valuable before losing it the next day or on the other hand making a stupid bet like all rich people do when they don't know what to do with their money. Maybe they should give it to the homeless instead of using it like Monopoly money.<br /><br />Or maybe this film will inspire you to help others. \n",
      "\n",
      "Label : \n",
      "pos\n"
     ]
    }
   ],
   "source": [
    "# Data Sample\n",
    "print('---- Data Sample ----')\n",
    "print('Input : ')\n",
    "print(' '.join(vars(train_data.examples[1])['text']),'\\n')\n",
    "print('Label : ')\n",
    "print(vars(train_data.examples[1])['label'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "pn4ddGIQLL_u"
   },
   "source": [
    "## Pre-processing Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 976,
     "status": "ok",
     "timestamp": 1593193212853,
     "user": {
      "displayName": "seongsu bang",
      "photoUrl": "",
      "userId": "06124410414614761416"
     },
     "user_tz": -540
    },
    "id": "gpIQqgb_G41G"
   },
   "outputs": [],
   "source": [
    "def PreProcessingText(input_sentence):\n",
    "    input_sentence = input_sentence.lower() # 소문자화\n",
    "    input_sentence = re.sub('<[^>]*>', repl= ' ', string = input_sentence) # \"<br />\" 처리\n",
    "    input_sentence = re.sub('[!\"#$%&\\()*+,-./:;<=>?@[\\\\]^_`{|}~]', repl= ' ', string = input_sentence) # 특수문자 처리 (\"'\" 제외)\n",
    "    input_sentence = re.sub('\\s+', repl= ' ', string = input_sentence) # 연속된 띄어쓰기 처리\n",
    "    if input_sentence:\n",
    "        return input_sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 6108,
     "status": "ok",
     "timestamp": 1593193227668,
     "user": {
      "displayName": "seongsu bang",
      "photoUrl": "",
      "userId": "06124410414614761416"
     },
     "user_tz": -540
    },
    "id": "5cIqIf34a2SR"
   },
   "outputs": [],
   "source": [
    "for example in train_data.examples:\n",
    "    vars(example)['text'] = PreProcessingText(' '.join(vars(example)['text'])).split()\n",
    "    \n",
    "for example in test_data.examples:\n",
    "    vars(example)['text'] = PreProcessingText(' '.join(vars(example)['text'])).split()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "iSucMR06LR8Y"
   },
   "source": [
    "## Making Vocab & Setting Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 901,
     "status": "ok",
     "timestamp": 1593193228570,
     "user": {
      "displayName": "seongsu bang",
      "photoUrl": "",
      "userId": "06124410414614761416"
     },
     "user_tz": -540
    },
    "id": "NBI0uTQUSbdt"
   },
   "outputs": [],
   "source": [
    "model_config = {'emb_type' : '', 'emb_dim' : 300}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 2903,
     "status": "ok",
     "timestamp": 1593193947508,
     "user": {
      "displayName": "seongsu bang",
      "photoUrl": "",
      "userId": "06124410414614761416"
     },
     "user_tz": -540
    },
    "id": "8LGSnQIsAHcv"
   },
   "outputs": [],
   "source": [
    "# no pre-trained \n",
    "TEXT.build_vocab(train_data,\n",
    "                 min_freq = 2, \n",
    "                 max_size = None,\n",
    "                 )\n",
    "\n",
    "LABEL.build_vocab(train_data)\n",
    "\n",
    "model_config['vocab_size'] = len(TEXT.vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 306
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 972,
     "status": "ok",
     "timestamp": 1593193950818,
     "user": {
      "displayName": "seongsu bang",
      "photoUrl": "",
      "userId": "06124410414614761416"
     },
     "user_tz": -540
    },
    "id": "vt7ulofTn6U4",
    "outputId": "6a5b9a8c-6f08-4097-d30a-29d0ab8ac582"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vocab Size : 51956\n",
      "Vocab Examples : \n",
      "\t [UNK] 0\n",
      "\t [PAD] 1\n",
      "\t the 2\n",
      "\t and 3\n",
      "\t a 4\n",
      "\t of 5\n",
      "\t to 6\n",
      "\t is 7\n",
      "\t in 8\n",
      "\t it 9\n",
      "---------------------------------\n",
      "Label Size : 2\n",
      "Lable Examples : \n",
      "\t neg 0\n",
      "\t pos 1\n"
     ]
    }
   ],
   "source": [
    "# Vocabulary Info\n",
    "print(f'Vocab Size : {len(TEXT.vocab)}')\n",
    "\n",
    "print('Vocab Examples : ')\n",
    "for idx, (k, v) in enumerate(TEXT.vocab.stoi.items()):\n",
    "    if idx >= 10:\n",
    "        break    \n",
    "    print('\\t', k, v)\n",
    "\n",
    "print('---------------------------------')\n",
    "\n",
    "# Label Info\n",
    "print(f'Label Size : {len(LABEL.vocab)}')\n",
    "\n",
    "print('Lable Examples : ')\n",
    "for idx, (k, v) in enumerate(LABEL.vocab.stoi.items()):\n",
    "    print('\\t', k, v)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "zuSh9SlQLfGp"
   },
   "source": [
    "## Spliting Validation Data & Making Data Iterator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1050,
     "status": "ok",
     "timestamp": 1593194358662,
     "user": {
      "displayName": "seongsu bang",
      "photoUrl": "",
      "userId": "06124410414614761416"
     },
     "user_tz": -540
    },
    "id": "CbH-Rha8___V"
   },
   "outputs": [],
   "source": [
    "# Spliting Valid set\n",
    "train_data, valid_data = train_data.split(random_state = random.seed(0),\n",
    "                                          split_ratio=0.8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1216,
     "status": "ok",
     "timestamp": 1593194359983,
     "user": {
      "displayName": "seongsu bang",
      "photoUrl": "",
      "userId": "06124410414614761416"
     },
     "user_tz": -540
    },
    "id": "LKOfVNAwAJLU"
   },
   "outputs": [],
   "source": [
    "model_config['batch_size'] = 30\n",
    "\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "\n",
    "train_iterator, valid_iterator, test_iterator = data.BucketIterator.splits(datasets=(train_data, valid_data, test_data), \n",
    "                                                                           batch_size=model_config['batch_size'],\n",
    "                                                                           device=device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# This part"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample_for_check = next(iter(train_iterator))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 238
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 61452,
     "status": "ok",
     "timestamp": 1592906338611,
     "user": {
      "displayName": "seongsu bang",
      "photoUrl": "",
      "userId": "06124410414614761416"
     },
     "user_tz": -540
    },
    "id": "8OFTE-xAwJYb",
    "outputId": "c58f837f-f1f2-44a2-f04e-77fdb9d571ea"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[torchtext.data.batch.Batch of size 30]\n",
      "\t[.text]:[torch.cuda.LongTensor of size 30x500 (GPU 0)]\n",
      "\t[.label]:[torch.cuda.FloatTensor of size 30 (GPU 0)]\n",
      "tensor([[   1,    1,    1,  ...,    2,  812,   96],\n",
      "        [   1,    1,    1,  ..., 1200,    5,   12],\n",
      "        [   1,    1,    1,  ...,   53,   98, 2187],\n",
      "        ...,\n",
      "        [   1,    1,    1,  ...,   10,  368,   78],\n",
      "        [   1,    1,    1,  ...,   19,    6,   64],\n",
      "        [   1,    1,    1,  ..., 2571,    5,   99]], device='cuda:0')\n",
      "tensor([1., 0., 1., 1., 0., 0., 1., 0., 0., 0., 0., 1., 1., 0., 0., 0., 1., 0.,\n",
      "        0., 0., 0., 0., 1., 1., 0., 0., 1., 1., 1., 1.], device='cuda:0')\n"
     ]
    }
   ],
   "source": [
    "# Check batch data\n",
    "sample_for_check = next(iter(train_iterator))\n",
    "print(sample_for_check)\n",
    "print(sample_for_check.text)\n",
    "print(sample_for_check.label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 71
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 61438,
     "status": "ok",
     "timestamp": 1592906338613,
     "user": {
      "displayName": "seongsu bang",
      "photoUrl": "",
      "userId": "06124410414614761416"
     },
     "user_tz": -540
    },
    "id": "NU1fESnXyR9E",
    "outputId": "6f981b3d-0a7c-401f-9fd8-939147741161"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ivan marco ricca and alexandre borges are partners in a company together with estevão george but the first ones contract a professional killer anisio to murder estevão the plot at least at the beginning doesn't explain very well why the guy does it and after receiving his money he starts blackmailing the two partners appearing in their company and saying he wants a job there as supervisor or something at the same time he meets marina mariana daughter of estevão and starts dating with her in a story like this where crime corruption betrayal and blackmail go hand in hand no one is innocent or can be victimized exception made to marina which is the only person who doesn't know what's going on and didn't betray anyone this film portrays with sarcasm the sad and cruel reality which exists in big metropolis like são paulo where crime is every day's presence we can feel irony but also veracity in characters like anisio brilliantly played by paulo miklos which does blackmail to the guys who paid him without any scruples and even flirts with the daughter of the guy he killed he really must be a monster to do something like that but of course i know there're people like him out there in brazil or any other place it's a very good movie cruel but truthful about a sad reality the acting is great and the soundtrack too\n",
      "pos\n"
     ]
    }
   ],
   "source": [
    "# Check reverting data\n",
    "print(' '.join([TEXT.vocab.itos[int(x)] for x in sample_for_check.text[0,:] if x not in [0,1]]))\n",
    "print(LABEL.vocab.itos[int(sample_for_check.label[0])]) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "cZXWakE7MxU8"
   },
   "source": [
    "## Making Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "CD2AK0NMAKr1"
   },
   "outputs": [],
   "source": [
    "class SentenceClassification(nn.Module):\n",
    "    def __init__(self, **model_config):\n",
    "        super(SentenceClassification, self).__init__()\n",
    "\n",
    "        if model_config['emb_type'] == 'glove' or 'fasttext':\n",
    "            self.emb = nn.Embedding(num_embeddings = model_config['vocab_size'],\n",
    "                                    embedding_dim = model_config['emb_dim'],\n",
    "                                    _weight = TEXT.vocab.vectors)\n",
    "        else:\n",
    "            self.emb = nn.Embedding(num_embeddings = model_config['vocab_size'],\n",
    "                                    embedding_dim = model_config['emb_dim'])\n",
    "        \n",
    "        self.bidirectional = model_config['bidirectional']\n",
    "        self.num_direction = 2 if model_config['bidirectional'] else 1\n",
    "        self.model_type = model_config['model_type'] \n",
    "\n",
    "        self.RNN = nn.RNN (input_size = model_config['emb_dim'],\n",
    "                           hidden_size = model_config['hidden_dim'],\n",
    "                           dropout = model_config['dropout'],\n",
    "                           bidirectional = model_config['bidirectional'],\n",
    "                           batch_first = model_config['batch_first'])\n",
    "        \n",
    "        self.LSTM= nn.LSTM(input_size = model_config['emb_dim'],\n",
    "                           hidden_size = model_config['hidden_dim'],\n",
    "                           dropout = model_config['dropout'],\n",
    "                           bidirectional = model_config['bidirectional'],\n",
    "                           batch_first = model_config['batch_first'])\n",
    "        \n",
    "        self.GRU = nn.GRU (input_size = model_config['emb_dim'],\n",
    "                           hidden_size = model_config['hidden_dim'],\n",
    "                           dropout = model_config['dropout'],\n",
    "                           bidirectional = model_config['bidirectional'],\n",
    "                           batch_first = model_config['batch_first'])\n",
    "    \n",
    "        self.fc = nn.Linear(model_config['hidden_dim'] * self.num_direction,\n",
    "                            model_config['output_dim'])\n",
    "        \n",
    "        self.drop = nn.Dropout(model_config['dropout'])\n",
    "\n",
    "    def forward(self, x):\n",
    "        \n",
    "        emb = self.emb(x) \n",
    "        # emb : (Batch_Size, Max_Seq_Length, Emb_dim)\n",
    "\n",
    "        if self.model_type == 'RNN':\n",
    "            output, hidden = self.RNN(emb) \n",
    "        elif self.model_type == 'LSTM':\n",
    "            output, (hidden, cell) = self.LSTM(emb)\n",
    "        elif self.model_type == 'GRU':\n",
    "            output, hidden = self.GRU(emb)\n",
    "        else:\n",
    "            raise NameError('Select model_type in [RNN, LSTM, GRU]')\n",
    "        \n",
    "        # output : (Batch_Size, Max_Seq_Length, Hidden_dim * num_direction) \n",
    "        # hidden : (num_direction, Batch_Size, Hidden_dim)\n",
    "        # hidden의 경우, batch_first 옵션이 안먹는 문제가 있음\n",
    "        \n",
    "        last_output = output[:,-1,:]\n",
    "\n",
    "        # last_output : (Batch_Size, Hidden_dim * num_direction)\n",
    "        return self.fc(self.drop(last_output))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "X_3DNQR3M30p"
   },
   "source": [
    "### Checking feed-forward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "r3VJ5BuHkm-4"
   },
   "outputs": [],
   "source": [
    "model_config.update(dict(batch_first = True,\n",
    "                         model_type = 'RNN',\n",
    "                         bidirectional = True,\n",
    "                         hidden_dim = 128,\n",
    "                         output_dim = 1,\n",
    "                         dropout = 0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "aWYjr7KQZxXu"
   },
   "outputs": [],
   "source": [
    "model = SentenceClassification(**model_config).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "uhGNULElZuI_"
   },
   "outputs": [],
   "source": [
    "predictions = model.forward(sample_for_check.text).squeeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "wabzuaD1AKjF"
   },
   "outputs": [],
   "source": [
    "loss_fn = nn.BCEWithLogitsLoss().to(device)\n",
    "\n",
    "def binary_accuracy(preds, y):\n",
    "    rounded_preds = torch.round(torch.sigmoid(preds))\n",
    "    correct = (rounded_preds == y).float() \n",
    "    acc = correct.sum()/len(correct)\n",
    "    return acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "HdOXGw9ze-lI"
   },
   "outputs": [],
   "source": [
    "loss = loss_fn(predictions, sample_for_check.label)\n",
    "acc = binary_accuracy(predictions, sample_for_check.label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 119
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 61680,
     "status": "ok",
     "timestamp": 1592906338921,
     "user": {
      "displayName": "seongsu bang",
      "photoUrl": "",
      "userId": "06124410414614761416"
     },
     "user_tz": -540
    },
    "id": "s5OBri4AsPVy",
    "outputId": "397eee3e-3b7d-470e-98b8-9eb21bf62c45"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([-0.0978,  0.3092, -0.2011, -0.6801,  0.4178,  0.2044,  0.1650,  0.2200,\n",
      "         0.4264,  0.2468,  0.3407,  0.0169, -0.3391,  0.0502, -0.2879, -0.1104,\n",
      "        -0.2063, -0.0021,  0.5160,  0.6616,  0.6315, -0.4962, -0.0985, -0.3059,\n",
      "        -0.3182, -0.1759, -0.3697,  0.4454, -0.3348,  0.5056], device='cuda:0',\n",
      "       grad_fn=<SqueezeBackward0>)\n",
      "tensor(0.7777, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>) tensor(0.3333, device='cuda:0')\n"
     ]
    }
   ],
   "source": [
    "print(predictions)\n",
    "print(loss, acc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "rswNBOAzoImC"
   },
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "WCgs1fdcAKXF"
   },
   "outputs": [],
   "source": [
    "def train(model, iterator, optimizer, loss_fn, idx_epoch, **model_params):\n",
    "    \n",
    "    epoch_loss = 0\n",
    "    epoch_acc = 0\n",
    "    \n",
    "    model.train() \n",
    "    batch_size = model_params['batch_size']\n",
    "\n",
    "    for idx, batch in enumerate(iterator):\n",
    "        \n",
    "        # Initializing\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        # Forward \n",
    "        predictions = model(batch.text).squeeze()\n",
    "        loss = loss_fn(predictions, batch.label)\n",
    "        acc = binary_accuracy(predictions, batch.label)\n",
    "        \n",
    "        sys.stdout.write(\n",
    "                    \"\\r\" + f\"[Train] Epoch : {idx_epoch:^3}\"\\\n",
    "                    f\"[{(idx + 1) * batch_size} / {len(iterator) * batch_size} ({100. * (idx + 1) / len(iterator) :.4}%)]\"\\\n",
    "                    f\"  Loss: {loss.item():.4}\"\\\n",
    "                    f\"  Acc : {acc.item():.4}\"\\\n",
    "                    )\n",
    "\n",
    "        # Backward \n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        # Update Epoch Performance\n",
    "        epoch_loss += loss.item()\n",
    "        epoch_acc += acc.item()\n",
    "        \n",
    "    return epoch_loss/len(iterator) , epoch_acc/len(iterator) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "3zqJ1gkFARwp"
   },
   "outputs": [],
   "source": [
    "def evaluate(model, iterator, loss_fn):\n",
    "    \n",
    "    epoch_loss = 0\n",
    "    epoch_acc = 0\n",
    "    \n",
    "    # evaluation mode\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        for batch in iterator:\n",
    "            predictions = model(batch.text).squeeze(1)\n",
    "            loss = loss_fn(predictions, batch.label)\n",
    "            acc = binary_accuracy(predictions, batch.label)\n",
    "\n",
    "            epoch_loss += loss.item()\n",
    "            epoch_acc += acc.item()\n",
    "        \n",
    "    return epoch_loss / len(iterator), epoch_acc / len(iterator)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "zIA_7QQzoLK1"
   },
   "source": [
    "### bi-RNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "n6GtVuVFUHqz"
   },
   "outputs": [],
   "source": [
    "model_config['model_type'] = 'RNN'\n",
    "model = SentenceClassification(**model_config).to(device)\n",
    "optimizer = torch.optim.Adam(model.parameters())\n",
    "loss_fn = nn.BCEWithLogitsLoss().to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 340
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 364107,
     "status": "ok",
     "timestamp": 1592906641386,
     "user": {
      "displayName": "seongsu bang",
      "photoUrl": "",
      "userId": "06124410414614761416"
     },
     "user_tz": -540
    },
    "id": "uTvlbDLnUHq2",
    "outputId": "3f3f3a88-890f-4e46-df4d-4abb24cbdf92"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------\n",
      "Model name : bi-RNN_\n",
      "---------------------------------\n",
      "[Train] Epoch :  0 [20010 / 20010 (100.0%)]  Loss: 0.532  Acc : 0.73337\n",
      "\t Saved at 0-epoch\n",
      "\t Epoch : 0 | Train Loss : 0.6426 | Train Acc : 0.6255\n",
      "\t Epoch : 0 | Valid Loss : 0.6249 | Valid Acc : 0.6546\n",
      "[Train] Epoch :  1 [20010 / 20010 (100.0%)]  Loss: 0.591  Acc : 0.77667\n",
      "\t Saved at 1-epoch\n",
      "\t Epoch : 1 | Train Loss : 0.6131 | Train Acc : 0.653\n",
      "\t Epoch : 1 | Valid Loss : 0.6069 | Valid Acc : 0.6637\n",
      "[Train] Epoch :  2 [20010 / 20010 (100.0%)]  Loss: 0.458  Acc : 0.88333\n",
      "\t Saved at 2-epoch\n",
      "\t Epoch : 2 | Train Loss : 0.5001 | Train Acc : 0.7583\n",
      "\t Epoch : 2 | Valid Loss : 0.5864 | Valid Acc : 0.6983\n",
      "[Train] Epoch :  3 [20010 / 20010 (100.0%)]  Loss: 0.3013  Acc : 0.8667\n",
      "\t Epoch : 3 | Train Loss : 0.4101 | Train Acc : 0.8204\n",
      "\t Epoch : 3 | Valid Loss : 0.6013 | Valid Acc : 0.713\n",
      "[Train] Epoch :  4 [20010 / 20010 (100.0%)]  Loss: 0.4681  Acc : 0.7333\n",
      "\t Epoch : 4 | Train Loss : 0.3619 | Train Acc : 0.8455\n",
      "\t Epoch : 4 | Valid Loss : 0.6166 | Valid Acc : 0.7134\n"
     ]
    }
   ],
   "source": [
    "N_EPOCH = 5\n",
    "\n",
    "best_valid_loss = float('inf')\n",
    "model_name = f\"{'bi-' if model_config['bidirectional'] else ''}{model_config['model_type']}_{model_config['emb_type']}\"\n",
    "\n",
    "print('---------------------------------')\n",
    "print(f'Model name : {model_name}')\n",
    "print('---------------------------------')\n",
    "\n",
    "for epoch in range(N_EPOCH):\n",
    "    train_loss, train_acc = train(model, train_iterator, optimizer, loss_fn, epoch, **model_config)\n",
    "    valid_loss, valid_acc = evaluate(model, valid_iterator, loss_fn)\n",
    "    print('')\n",
    "    if valid_loss < best_valid_loss:\n",
    "        best_valid_loss = valid_loss\n",
    "        torch.save(model.state_dict(), f'./{model_name}.pt')\n",
    "        print(f'\\t Saved at {epoch}-epoch')\n",
    "\n",
    "    print(f'\\t Epoch : {epoch} | Train Loss : {train_loss:.4} | Train Acc : {train_acc:.4}')\n",
    "    print(f'\\t Epoch : {epoch} | Valid Loss : {valid_loss:.4} | Valid Acc : {valid_acc:.4}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 394038,
     "status": "ok",
     "timestamp": 1592906671343,
     "user": {
      "displayName": "seongsu bang",
      "photoUrl": "",
      "userId": "06124410414614761416"
     },
     "user_tz": -540
    },
    "id": "akDRWiykUHq5",
    "outputId": "da834104-d34f-4c5d-f009-78905305042a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Loss : 0.585 | Test Acc : 0.7046\n"
     ]
    }
   ],
   "source": [
    "# Test set\n",
    "model.load_state_dict(torch.load(f'./{model_name}.pt'))\n",
    "test_loss, test_acc = evaluate(model, test_iterator, loss_fn)\n",
    "print(f'Test Loss : {test_loss:.4} | Test Acc : {test_acc:.4}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "uxG4rgwTo8D7"
   },
   "source": [
    "### bi-LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "wpW7ui5mUBTk"
   },
   "outputs": [],
   "source": [
    "model_config['model_type'] = 'LSTM'\n",
    "model = SentenceClassification(**model_config).to(device)\n",
    "optimizer = torch.optim.Adam(model.parameters())\n",
    "loss_fn = nn.BCEWithLogitsLoss().to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 374
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 780775,
     "status": "ok",
     "timestamp": 1592907058102,
     "user": {
      "displayName": "seongsu bang",
      "photoUrl": "",
      "userId": "06124410414614761416"
     },
     "user_tz": -540
    },
    "id": "CDN-nLxJUBTo",
    "outputId": "d2a2703c-d56b-42e7-f858-9ccdb1481325"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------\n",
      "Model name : bi-LSTM_\n",
      "---------------------------------\n",
      "[Train] Epoch :  0 [20010 / 20010 (100.0%)]  Loss: 0.388  Acc : 0.86673\n",
      "\t Saved at 0-epoch\n",
      "\t Epoch : 0 | Train Loss : 0.5992 | Train Acc : 0.6719\n",
      "\t Epoch : 0 | Valid Loss : 0.4954 | Valid Acc : 0.7604\n",
      "[Train] Epoch :  1 [20010 / 20010 (100.0%)]  Loss: 0.3265  Acc : 0.8667\n",
      "\t Saved at 1-epoch\n",
      "\t Epoch : 1 | Train Loss : 0.4223 | Train Acc : 0.8127\n",
      "\t Epoch : 1 | Valid Loss : 0.4332 | Valid Acc : 0.8202\n",
      "[Train] Epoch :  2 [20010 / 20010 (100.0%)]  Loss: 0.4906  Acc : 0.7667\n",
      "\t Saved at 2-epoch\n",
      "\t Epoch : 2 | Train Loss : 0.2878 | Train Acc : 0.8832\n",
      "\t Epoch : 2 | Valid Loss : 0.4002 | Valid Acc : 0.8396\n",
      "[Train] Epoch :  3 [20010 / 20010 (100.0%)]  Loss: 0.1404  Acc : 0.93337\n",
      "\t Epoch : 3 | Train Loss : 0.171 | Train Acc : 0.935\n",
      "\t Epoch : 3 | Valid Loss : 0.418 | Valid Acc : 0.8508\n",
      "[Train] Epoch :  4 [20010 / 20010 (100.0%)]  Loss: 0.2688  Acc : 0.93337\n",
      "\t Epoch : 4 | Train Loss : 0.09108 | Train Acc : 0.9692\n",
      "\t Epoch : 4 | Valid Loss : 0.481 | Valid Acc : 0.8526\n"
     ]
    }
   ],
   "source": [
    "N_EPOCH = 5\n",
    "\n",
    "best_valid_loss = float('inf')\n",
    "model_name = f\"{'bi-' if model_config['bidirectional'] else ''}{model_config['model_type']}_{model_config['emb_type']}\"\n",
    "\n",
    "print('---------------------------------')\n",
    "print(f'Model name : {model_name}')\n",
    "print('---------------------------------')\n",
    "\n",
    "for epoch in range(N_EPOCH):\n",
    "    train_loss, train_acc = train(model, train_iterator, optimizer, loss_fn, epoch, **model_config)\n",
    "    valid_loss, valid_acc = evaluate(model, valid_iterator, loss_fn)\n",
    "    print('')\n",
    "    if valid_loss < best_valid_loss:\n",
    "        best_valid_loss = valid_loss\n",
    "        torch.save(model.state_dict(), f'./{model_name}.pt')\n",
    "        print(f'\\t Saved at {epoch}-epoch')\n",
    "\n",
    "    print(f'\\t Epoch : {epoch} | Train Loss : {train_loss:.4} | Train Acc : {train_acc:.4}')\n",
    "    print(f'\\t Epoch : {epoch} | Valid Loss : {valid_loss:.4} | Valid Acc : {valid_acc:.4}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 810789,
     "status": "ok",
     "timestamp": 1592907088133,
     "user": {
      "displayName": "seongsu bang",
      "photoUrl": "",
      "userId": "06124410414614761416"
     },
     "user_tz": -540
    },
    "id": "d8qjmPU7UBTq",
    "outputId": "70d0f044-f4d0-4b5c-fcfb-f97e418fc4e0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Loss : 0.4053 | Test Acc : 0.8325\n"
     ]
    }
   ],
   "source": [
    "# Test set\n",
    "model.load_state_dict(torch.load(f'./{model_name}.pt'))\n",
    "test_loss, test_acc = evaluate(model, test_iterator, loss_fn)\n",
    "print(f'Test Loss : {test_loss:.4} | Test Acc : {test_acc:.4}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "FfDGBCWRxnB_"
   },
   "source": [
    "### bi-GRU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "khyItx4zxnCA"
   },
   "outputs": [],
   "source": [
    "model_config['model_type'] = 'GRU'\n",
    "model = SentenceClassification(**model_config).to(device)\n",
    "optimizer = torch.optim.Adam(model.parameters())\n",
    "loss_fn = nn.BCEWithLogitsLoss().to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 357
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1191798,
     "status": "ok",
     "timestamp": 1592907469165,
     "user": {
      "displayName": "seongsu bang",
      "photoUrl": "",
      "userId": "06124410414614761416"
     },
     "user_tz": -540
    },
    "id": "om_7UjicxnCD",
    "outputId": "18750447-423e-4d24-f022-113d735fe6cc"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------\n",
      "Model name : bi-GRU_\n",
      "---------------------------------\n",
      "[Train] Epoch :  0 [20010 / 20010 (100.0%)]  Loss: 0.4023  Acc : 0.9667\n",
      "\t Saved at 0-epoch\n",
      "\t Epoch : 0 | Train Loss : 0.5662 | Train Acc : 0.7022\n",
      "\t Epoch : 0 | Valid Loss : 0.4814 | Valid Acc : 0.7778\n",
      "[Train] Epoch :  1 [20010 / 20010 (100.0%)]  Loss: 0.4055  Acc : 0.8333\n",
      "\t Saved at 1-epoch\n",
      "\t Epoch : 1 | Train Loss : 0.4068 | Train Acc : 0.8222\n",
      "\t Epoch : 1 | Valid Loss : 0.4136 | Valid Acc : 0.829\n",
      "[Train] Epoch :  2 [20010 / 20010 (100.0%)]  Loss: 0.2221  Acc : 0.8667\n",
      "\t Saved at 2-epoch\n",
      "\t Epoch : 2 | Train Loss : 0.3153 | Train Acc : 0.8643\n",
      "\t Epoch : 2 | Valid Loss : 0.3651 | Valid Acc : 0.8562\n",
      "[Train] Epoch :  3 [20010 / 20010 (100.0%)]  Loss: 0.3287  Acc : 0.90673\n",
      "\t Saved at 3-epoch\n",
      "\t Epoch : 3 | Train Loss : 0.1717 | Train Acc : 0.9361\n",
      "\t Epoch : 3 | Valid Loss : 0.3541 | Valid Acc : 0.8615\n",
      "[Train] Epoch :  4 [20010 / 20010 (100.0%)]  Loss: 0.05094  Acc : 1.0667\n",
      "\t Epoch : 4 | Train Loss : 0.07654 | Train Acc : 0.9768\n",
      "\t Epoch : 4 | Valid Loss : 0.4239 | Valid Acc : 0.8784\n"
     ]
    }
   ],
   "source": [
    "N_EPOCH = 5\n",
    "\n",
    "best_valid_loss = float('inf')\n",
    "model_name = f\"{'bi-' if model_config['bidirectional'] else ''}{model_config['model_type']}_{model_config['emb_type']}\"\n",
    "\n",
    "print('---------------------------------')\n",
    "print(f'Model name : {model_name}')\n",
    "print('---------------------------------')\n",
    "\n",
    "for epoch in range(N_EPOCH):\n",
    "    train_loss, train_acc = train(model, train_iterator, optimizer, loss_fn, epoch, **model_config)\n",
    "    valid_loss, valid_acc = evaluate(model, valid_iterator, loss_fn)\n",
    "    print('')\n",
    "    if valid_loss < best_valid_loss:\n",
    "        best_valid_loss = valid_loss\n",
    "        torch.save(model.state_dict(), f'./{model_name}.pt')\n",
    "        print(f'\\t Saved at {epoch}-epoch')\n",
    "\n",
    "    print(f'\\t Epoch : {epoch} | Train Loss : {train_loss:.4} | Train Acc : {train_acc:.4}')\n",
    "    print(f'\\t Epoch : {epoch} | Valid Loss : {valid_loss:.4} | Valid Acc : {valid_acc:.4}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1222229,
     "status": "ok",
     "timestamp": 1592907499601,
     "user": {
      "displayName": "seongsu bang",
      "photoUrl": "",
      "userId": "06124410414614761416"
     },
     "user_tz": -540
    },
    "id": "ZOL9MoBWxnCG",
    "outputId": "13add090-697c-4acb-fa54-285ca1746fbe"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Loss : 0.3809 | Test Acc : 0.8504\n"
     ]
    }
   ],
   "source": [
    "# Test set\n",
    "model.load_state_dict(torch.load(f'./{model_name}.pt'))\n",
    "test_loss, test_acc = evaluate(model, test_iterator, loss_fn)\n",
    "print(f'Test Loss : {test_loss:.4} | Test Acc : {test_acc:.4}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "GEWK8wnkuEzA"
   },
   "source": [
    "## Inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1222227,
     "status": "ok",
     "timestamp": 1592907499602,
     "user": {
      "displayName": "seongsu bang",
      "photoUrl": "",
      "userId": "06124410414614761416"
     },
     "user_tz": -540
    },
    "id": "x5VKyllQuReq",
    "outputId": "d888c44d-7dda-4c1b-b284-dcbd41458ffc"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_config['model_type'] = 'GRU'\n",
    "model = SentenceClassification(**model_config).to(device)\n",
    "model.load_state_dict(torch.load(f\"./{'bi-' if model_config['bidirectional'] else ''}{model_config['model_type']}_{model_config['emb_type']}.pt\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "MPifO6k5ugGG"
   },
   "outputs": [],
   "source": [
    "def predict_sentiment(model, sentence):\n",
    "    model.eval()\n",
    "    indexed = TEXT.numericalize(TEXT.pad([TEXT.tokenize(PreProcessingText(sentence))]))\n",
    "    input_data = torch.LongTensor(indexed).to(device)\n",
    "    prediction = torch.sigmoid(model(input_data))\n",
    "    return prediction.item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1222223,
     "status": "ok",
     "timestamp": 1592907499604,
     "user": {
      "displayName": "seongsu bang",
      "photoUrl": "",
      "userId": "06124410414614761416"
     },
     "user_tz": -540
    },
    "id": "gY4lDbr-xB7V",
    "outputId": "e691632a-5c3c-4e20-a90d-5d84762a8c3d"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8596924543380737"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_sentence = 'this movie is FUN'\n",
    "predict_sentiment(model = model, sentence = test_sentence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "authorship_tag": "ABX9TyOye5Dt4Q3CTveRWxRMC3G8",
   "collapsed_sections": [],
   "name": "6-2_model_imdb_scratch.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}