Prediction_with_augmented_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#coding=utf-8\n",
    "import os\n",
    "import yaml\n",
    "import argparse\n",
    "from datetime import datetime\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import torch\n",
    "from torch.autograd import Variable\n",
    "import torch.nn as nn\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "# utils\n",
    "from utils import get_embedding, load_embed, save_embed, data_preprocessing\n",
    "# data\n",
    "from data import myDS, mytestDS\n",
    "# model\n",
    "from model import Siamese_lstm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = {\n",
    "    'experiment_name': 'siamese-baseline-aug-1000-less',\n",
    "    'task': 'train',\n",
    "    'make_dict': True,\n",
    "    'data_preprocessing': False,\n",
    "\n",
    "    'ckpt_dir': 'ckpt/',\n",
    "\n",
    "    'training':{\n",
    "        'num_epochs': 20,\n",
    "        'learning_rate': 0.01,\n",
    "        'optimizer': 'sgd'\n",
    "    },\n",
    "    \n",
    "    'embedding':{\n",
    "        'full_embedding_path': 'input/wiki.es.vec',\n",
    "        'cur_embedding_path': 'input/embedding_aug.pkl',\n",
    "    },\n",
    "        \n",
    "    'model':{\n",
    "        'fc_dropout': 0.1,\n",
    "        'fc_dim': 100,\n",
    "        'name': 'siamese',\n",
    "        'embed_size': 300,\n",
    "        'batch_size': 1,\n",
    "        'embedding_freeze': False,\n",
    "        'encoder':{\n",
    "            'hidden_size': 150,\n",
    "            'num_layers': 1,\n",
    "            'bidirectional': False,\n",
    "            'dropout': 0.0,\n",
    "        },  \n",
    "    },   \n",
    "    \n",
    "    'result':{\n",
    "        'filename':'result.txt',\n",
    "        'filepath':'res/',\n",
    "    }\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" Read Data \"\"\"\n",
    "\n",
    "train_data = pd.read_csv('input/cleaned_train.csv')\n",
    "test_data = pd.read_csv('input/cleaned_test.csv')\n",
    "\n",
    "aug_data = pd.read_csv('input/aug_sp_1000.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils import removeSpanishStopWords, cleanSpanish, clean_sent\n",
    "aug_data = aug_data.drop(['Unnamed: 0'], axis=1)\n",
    "aug_data.columns = ['spanish1', 'spanish2', 'result']\n",
    "cleanSpanish(aug_data)\n",
    "from nltk.corpus import stopwords\n",
    "stops1 = set(stopwords.words(\"spanish\"))\n",
    "removeSpanishStopWords(aug_data, stops1)\n",
    "aug_data.replace('', np.nan, inplace=True)\n",
    "dirty_data = aug_data[aug_data.isnull().any(axis=1)]\n",
    "print('dirty sample count:', dirty_data.shape[0])\n",
    "print('positive dirty training sample:', len(dirty_data[dirty_data['result'] == 1]))\n",
    "print('negative dirty training sample:', len(dirty_data[dirty_data['result'] == 0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Aug Neg: 38345\n",
      "Aug Pos: 28635\n"
     ]
    }
   ],
   "source": [
    "print('Aug Neg:', aug_data[aug_data['result'] == 0].shape[0])\n",
    "print('Aug Pos:', aug_data[aug_data['result'] == 1].shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "aug_data.columns = ['s1','s2','label']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = pd.concat([train_data, aug_data]).reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = train_data.drop(['index'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data.columns = ['s1', 's2']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>s1</th>\n",
       "      <th>s2</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>hola hago clic producto recibido</td>\n",
       "      <td>compré producto recibido correo electrónico co...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>hola cerré disputa 21 mayo 2017 dice realizará...</td>\n",
       "      <td>obtuve reembolso dinero pasado dos meses cuánd...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ordené españa españa ahora mandan pedido china</td>\n",
       "      <td>pedido llegó color diferente pedí</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>debo pagar impuestos personalizados</td>\n",
       "      <td>cómo pagar derechos aduana</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>recibí pedido</td>\n",
       "      <td>pedido muestra pagado hice</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  s1  \\\n",
       "0                   hola hago clic producto recibido   \n",
       "1  hola cerré disputa 21 mayo 2017 dice realizará...   \n",
       "2     ordené españa españa ahora mandan pedido china   \n",
       "3                debo pagar impuestos personalizados   \n",
       "4                                      recibí pedido   \n",
       "\n",
       "                                                  s2  label  \n",
       "0  compré producto recibido correo electrónico co...      0  \n",
       "1  obtuve reembolso dinero pasado dos meses cuánd...      0  \n",
       "2                  pedido llegó color diferente pedí      0  \n",
       "3                         cómo pagar derechos aduana      1  \n",
       "4                         pedido muestra pagado hice      0  "
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>s1</th>\n",
       "      <th>s2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>cómo puedo recibir reembolso mediante tarjeta</td>\n",
       "      <td>cómo puedo recibir reembolso</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>cómo puedo recibir reembolso mediante tarjeta</td>\n",
       "      <td>cómo puedo recibir reembolso si banco ido banc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>podido pagar tarjeta debo hacer</td>\n",
       "      <td>puedo pagar tarjeta débito puedo hacer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>podido pagar tarjeta debo hacer</td>\n",
       "      <td>puedo pagar pedido tarjeta visa débito puedo h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>podido pagar tarjeta debo hacer</td>\n",
       "      <td>puedo pagar tarjeta</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              s1  \\\n",
       "0  cómo puedo recibir reembolso mediante tarjeta   \n",
       "1  cómo puedo recibir reembolso mediante tarjeta   \n",
       "2                podido pagar tarjeta debo hacer   \n",
       "3                podido pagar tarjeta debo hacer   \n",
       "4                podido pagar tarjeta debo hacer   \n",
       "\n",
       "                                                  s2  \n",
       "0                       cómo puedo recibir reembolso  \n",
       "1  cómo puedo recibir reembolso si banco ido banc...  \n",
       "2             puedo pagar tarjeta débito puedo hacer  \n",
       "3  puedo pagar pedido tarjeta visa débito puedo h...  \n",
       "4                                puedo pagar tarjeta  "
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data size: 88307 5000\n"
     ]
    }
   ],
   "source": [
    "# split dataset\n",
    "msk = np.random.rand(len(train_data)) < 0.8\n",
    "train = train_data[msk]\n",
    "valid = train_data[~msk]\n",
    "all_sents = train_data['s1'].tolist() + train_data['s2'].tolist() + test_data['s1'].tolist() + test_data['s2'].tolist()\n",
    "\n",
    "# dataset\n",
    "trainDS = myDS(train, all_sents)\n",
    "validDS = myDS(valid, all_sents)\n",
    "\n",
    "print('Data size:',train_data.shape[0], test_data.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(88307, 3)"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "33933 54374 0.3842617233061932\n"
     ]
    }
   ],
   "source": [
    "po = train_data[train_data['label']==1].shape[0]\n",
    "ne = train_data[train_data['label']==0].shape[0]\n",
    "print(po, ne, po/train_data.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Making embedding...\n",
      "Found 5189/5819 words with embedding vectors\n",
      "Missing Ratio: 10.83%\n",
      "Filled missing words' embeddings.\n",
      "Embedding Matrix Size:  5819\n",
      "Embedding saved\n",
      "Saved generated embedding.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "full_embed_path = config['embedding']['full_embedding_path']\n",
    "cur_embed_path = config['embedding']['cur_embedding_path']\n",
    "\n",
    "if os.path.exists(cur_embed_path) and not config['make_dict']:\n",
    "    embed_dict = load_embed(cur_embed_path)\n",
    "    print('Loaded existing embedding.')\n",
    "else:\n",
    "    print('Making embedding...')\n",
    "    embed_dict = get_embedding(trainDS.vocab._id2word, full_embed_path)\n",
    "    save_embed(embed_dict,cur_embed_path)\n",
    "    print('Saved generated embedding.')\n",
    "\n",
    "\n",
    "vocab_size = len(embed_dict)\n",
    "# initialize nn embedding\n",
    "embedding = nn.Embedding(vocab_size, config['model']['embed_size'])\n",
    "embed_list = []\n",
    "for word in trainDS.vocab._id2word:\n",
    "    embed_list.append(embed_dict[word])\n",
    "weight_matrix = np.array(embed_list)\n",
    "# pass weights to nn embedding\n",
    "embedding.weight = nn.Parameter(torch.from_numpy(weight_matrix).type(torch.FloatTensor), requires_grad = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Siamese_lstm(\n",
      "  (encoder): LSTMEncoder(\n",
      "    (embedding): Embedding(5819, 300)\n",
      "    (lstm): LSTM(300, 150)\n",
      "  )\n",
      "  (classifier): Sequential(\n",
      "    (0): Dropout(p=0.1)\n",
      "    (1): Linear(in_features=600, out_features=100, bias=True)\n",
      "    (2): Tanh()\n",
      "    (3): Dropout(p=0.1)\n",
      "    (4): Linear(in_features=100, out_features=2, bias=True)\n",
      "  )\n",
      ")\n",
      "Optimizer: sgd\n",
      "Learning rate: 0.01\n",
      "Fresh start!\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# embedding\n",
    "config['embedding_matrix'] = embedding\n",
    "config['vocab_size'] = len(embed_dict)\n",
    "\n",
    "# model\n",
    "siamese = Siamese_lstm(config)\n",
    "print(siamese)\n",
    "\n",
    "# loss func\n",
    "loss_weights = Variable(torch.FloatTensor([1, 3]))\n",
    "if torch.cuda.is_available():\n",
    "    loss_weights = loss_weights.cuda()\n",
    "criterion = torch.nn.CrossEntropyLoss(loss_weights)\n",
    "\n",
    "# optimizer\n",
    "learning_rate = config['training']['learning_rate']\n",
    "if config['training']['optimizer'] == 'sgd':\n",
    "    optimizer = torch.optim.SGD(filter(lambda x: x.requires_grad, siamese.parameters()), lr=learning_rate)\n",
    "elif config['training']['optimizer'] == 'adam':\n",
    "    optimizer = torch.optim.Adam(filter(lambda x: x.requires_grad, siamese.parameters()), lr=learning_rate)\n",
    "elif config['training']['optimizer'] == 'adadelta':\n",
    "    optimizer = torch.optim.Adadelta(filter(lambda x: x.requires_grad, siamese.parameters()), lr=learning_rate)\n",
    "elif config['training']['optimizer'] == 'rmsprop':\n",
    "    optimizer = torch.optim.RMSprop(filter(lambda x: x.requires_grad, siamese.parameters()), lr=learning_rate)\n",
    "print('Optimizer:', config['training']['optimizer'])\n",
    "print('Learning rate:', config['training']['learning_rate'])\n",
    "\n",
    "# log info\n",
    "train_log_string = '%s :: Epoch %i :: Iter %i / %i :: train loss: %0.4f'\n",
    "valid_log_string = '%s :: Epoch %i :: valid loss: %0.4f\\n'\n",
    "\n",
    "# Restore saved model (if one exists).\n",
    "ckpt_path = os.path.join(config['ckpt_dir'], config['experiment_name']+'.pt')\n",
    "\n",
    "if os.path.exists(ckpt_path):\n",
    "    print('Loading checkpoint: %s' % ckpt_path)\n",
    "    ckpt = torch.load(ckpt_path)\n",
    "    epoch = ckpt['epoch']\n",
    "    siamese.load_state_dict(ckpt['siamese'])\n",
    "    optimizer.load_state_dict(ckpt['optimizer'])\n",
    "else:\n",
    "    epoch = 1\n",
    "    print('Fresh start!\\n')\n",
    "\n",
    "\n",
    "if torch.cuda.is_available():\n",
    "    criterion = criterion.cuda()\n",
    "    siamese = siamese.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Experiment: siamese-baseline-aug-1000\n",
      "\n",
      "Start Epoch 1 Training...\n",
      "2018-07-29 21:45:20.762607 :: Epoch 1 :: Iter 5000 / 70298 :: train loss: 0.4657\n",
      "2018-07-29 21:47:02.258746 :: Epoch 1 :: Iter 10000 / 70298 :: train loss: 0.2669\n",
      "2018-07-29 21:48:43.323065 :: Epoch 1 :: Iter 15000 / 70298 :: train loss: 0.2125\n",
      "2018-07-29 21:50:25.156432 :: Epoch 1 :: Iter 20000 / 70298 :: train loss: 0.2049\n",
      "2018-07-29 21:52:06.390792 :: Epoch 1 :: Iter 25000 / 70298 :: train loss: 0.1875\n",
      "2018-07-29 21:53:48.405431 :: Epoch 1 :: Iter 30000 / 70298 :: train loss: 0.1781\n",
      "2018-07-29 21:55:30.768276 :: Epoch 1 :: Iter 35000 / 70298 :: train loss: 0.1779\n",
      "2018-07-29 21:57:12.911882 :: Epoch 1 :: Iter 40000 / 70298 :: train loss: 0.1559\n",
      "2018-07-29 21:58:56.050493 :: Epoch 1 :: Iter 45000 / 70298 :: train loss: 0.1536\n",
      "2018-07-29 22:00:39.907874 :: Epoch 1 :: Iter 50000 / 70298 :: train loss: 0.1529\n",
      "2018-07-29 22:02:23.451565 :: Epoch 1 :: Iter 55000 / 70298 :: train loss: 0.1425\n",
      "2018-07-29 22:04:11.152360 :: Epoch 1 :: Iter 60000 / 70298 :: train loss: 0.1462\n",
      "2018-07-29 22:06:08.046836 :: Epoch 1 :: Iter 65000 / 70298 :: train loss: 0.1497\n",
      "2018-07-29 22:08:06.657585 :: Epoch 1 :: Iter 70000 / 70298 :: train loss: 0.1505\n",
      "Train Loss at epoch 1: 0.19599385559558868\n",
      "\n",
      "Epoch 1 Validating...\n",
      "2018-07-29 22:10:26.501255 :: Epoch 1 :: valid loss: 0.1305\n",
      "\n",
      "Model saved!\n",
      "\n",
      "Start Epoch 2 Training...\n",
      "2018-07-29 22:12:21.537629 :: Epoch 2 :: Iter 5000 / 70298 :: train loss: 0.1387\n",
      "2018-07-29 22:14:17.019737 :: Epoch 2 :: Iter 10000 / 70298 :: train loss: 0.1358\n",
      "2018-07-29 22:16:12.215868 :: Epoch 2 :: Iter 15000 / 70298 :: train loss: 0.1330\n",
      "2018-07-29 22:18:06.121206 :: Epoch 2 :: Iter 20000 / 70298 :: train loss: 0.1305\n",
      "2018-07-29 22:19:52.318495 :: Epoch 2 :: Iter 25000 / 70298 :: train loss: 0.1221\n",
      "2018-07-29 22:21:36.701552 :: Epoch 2 :: Iter 30000 / 70298 :: train loss: 0.1226\n",
      "2018-07-29 22:23:21.045632 :: Epoch 2 :: Iter 35000 / 70298 :: train loss: 0.1202\n",
      "2018-07-29 22:25:04.582988 :: Epoch 2 :: Iter 40000 / 70298 :: train loss: 0.1232\n",
      "2018-07-29 22:26:48.414237 :: Epoch 2 :: Iter 45000 / 70298 :: train loss: 0.1273\n",
      "2018-07-29 22:28:32.582685 :: Epoch 2 :: Iter 50000 / 70298 :: train loss: 0.1239\n",
      "2018-07-29 22:30:16.663045 :: Epoch 2 :: Iter 55000 / 70298 :: train loss: 0.1207\n",
      "2018-07-29 22:32:00.379194 :: Epoch 2 :: Iter 60000 / 70298 :: train loss: 0.1273\n",
      "2018-07-29 22:33:44.793118 :: Epoch 2 :: Iter 65000 / 70298 :: train loss: 0.1275\n",
      "2018-07-29 22:35:28.211524 :: Epoch 2 :: Iter 70000 / 70298 :: train loss: 0.1216\n",
      "Train Loss at epoch 2: 0.12666106224060059\n",
      "\n",
      "Epoch 2 Validating...\n",
      "2018-07-29 22:37:27.172977 :: Epoch 2 :: valid loss: 0.1178\n",
      "\n",
      "Model saved!\n",
      "\n",
      "Start Epoch 3 Training...\n",
      "2018-07-29 22:39:12.701448 :: Epoch 3 :: Iter 5000 / 70298 :: train loss: 0.1129\n",
      "2018-07-29 22:40:59.598545 :: Epoch 3 :: Iter 10000 / 70298 :: train loss: 0.0970\n",
      "2018-07-29 22:42:42.865417 :: Epoch 3 :: Iter 15000 / 70298 :: train loss: 0.1051\n",
      "2018-07-29 22:44:28.131768 :: Epoch 3 :: Iter 20000 / 70298 :: train loss: 0.1191\n",
      "2018-07-29 22:46:16.013541 :: Epoch 3 :: Iter 25000 / 70298 :: train loss: 0.1208\n",
      "2018-07-29 22:48:01.702123 :: Epoch 3 :: Iter 30000 / 70298 :: train loss: 0.1093\n",
      "2018-07-29 22:49:44.684525 :: Epoch 3 :: Iter 35000 / 70298 :: train loss: 0.1087\n",
      "2018-07-29 22:51:26.760079 :: Epoch 3 :: Iter 40000 / 70298 :: train loss: 0.1137\n",
      "2018-07-29 22:53:12.678090 :: Epoch 3 :: Iter 45000 / 70298 :: train loss: 0.1001\n",
      "2018-07-29 22:55:01.092315 :: Epoch 3 :: Iter 50000 / 70298 :: train loss: 0.0954\n",
      "2018-07-29 22:56:44.575538 :: Epoch 3 :: Iter 55000 / 70298 :: train loss: 0.1032\n",
      "2018-07-29 22:58:26.263879 :: Epoch 3 :: Iter 60000 / 70298 :: train loss: 0.1105\n",
      "2018-07-29 23:00:11.976932 :: Epoch 3 :: Iter 65000 / 70298 :: train loss: 0.1103\n",
      "2018-07-29 23:01:58.993819 :: Epoch 3 :: Iter 70000 / 70298 :: train loss: 0.1092\n",
      "Train Loss at epoch 3: 0.10821811854839325\n",
      "\n",
      "Epoch 3 Validating...\n",
      "2018-07-29 23:04:04.802974 :: Epoch 3 :: valid loss: 0.1136\n",
      "\n",
      "Model saved!\n",
      "\n",
      "Start Epoch 4 Training...\n",
      "2018-07-29 23:05:48.516490 :: Epoch 4 :: Iter 5000 / 70298 :: train loss: 0.0869\n",
      "2018-07-29 23:07:32.798279 :: Epoch 4 :: Iter 10000 / 70298 :: train loss: 0.0891\n",
      "2018-07-29 23:09:17.980371 :: Epoch 4 :: Iter 15000 / 70298 :: train loss: 0.0938\n",
      "2018-07-29 23:11:04.218696 :: Epoch 4 :: Iter 20000 / 70298 :: train loss: 0.1041\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-127-b8004c63df11>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     30\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     31\u001b[0m             \u001b[0;31m# input\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 32\u001b[0;31m             \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msiamese\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ms2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     33\u001b[0m             \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moutput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     34\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    475\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    476\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 477\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    478\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    479\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/PycharmProjects/LSTM-siamese/model.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, s1, s2)\u001b[0m\n\u001b[1;32m     77\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0minput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0ms2\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mj\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     78\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 79\u001b[0;31m             \u001b[0mv2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mh2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mc2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mencoder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mh2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mc2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     80\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     81\u001b[0m         \u001b[0;31m# utilize these two encoded vectors\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    475\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    476\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 477\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    478\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    479\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/PycharmProjects/LSTM-siamese/model.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input, hidden, cell)\u001b[0m\n\u001b[1;32m     32\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhidden\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcell\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     33\u001b[0m         \u001b[0minput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0membedding\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 34\u001b[0;31m         \u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhidden\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcell\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlstm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhidden\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcell\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     35\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhidden\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcell\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     36\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    475\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    476\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 477\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    478\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    479\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/torch/nn/modules/rnn.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input, hx)\u001b[0m\n\u001b[1;32m    190\u001b[0m             \u001b[0mflat_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mflat_weight\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    191\u001b[0m         )\n\u001b[0;32m--> 192\u001b[0;31m         \u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhidden\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mall_weights\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_sizes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    193\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mis_packed\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    194\u001b[0m             \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mPackedSequence\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_sizes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/torch/nn/_functions/rnn.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(input, *fargs, **fkwargs)\u001b[0m\n\u001b[1;32m    322\u001b[0m             \u001b[0mfunc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdecorator\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    323\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 324\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mfargs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mfkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    325\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    326\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/torch/nn/_functions/rnn.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(input, weight, hidden, batch_sizes)\u001b[0m\n\u001b[1;32m    242\u001b[0m             \u001b[0minput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtranspose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    243\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 244\u001b[0;31m         \u001b[0mnexth\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhidden\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_sizes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    245\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    246\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mbatch_first\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mvariable_length\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/torch/nn/_functions/rnn.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(input, hidden, weight, batch_sizes)\u001b[0m\n\u001b[1;32m     85\u001b[0m                 \u001b[0ml\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mi\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mnum_directions\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mj\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     86\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 87\u001b[0;31m                 \u001b[0mhy\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minner\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhidden\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0ml\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0ml\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_sizes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     88\u001b[0m                 \u001b[0mnext_hidden\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhy\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     89\u001b[0m                 \u001b[0mall_output\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/torch/nn/_functions/rnn.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(input, hidden, weight, batch_sizes)\u001b[0m\n\u001b[1;32m    114\u001b[0m         \u001b[0msteps\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mreverse\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    115\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0msteps\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 116\u001b[0;31m             \u001b[0mhidden\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minner\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhidden\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    117\u001b[0m             \u001b[0;31m# hack to handle LSTM\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    118\u001b[0m             \u001b[0moutput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhidden\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhidden\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mhidden\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/torch/nn/_functions/rnn.py\u001b[0m in \u001b[0;36mLSTMCell\u001b[0;34m(input, hidden, w_ih, w_hh, b_ih, b_hh)\u001b[0m\n\u001b[1;32m     32\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     33\u001b[0m     \u001b[0mhx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhidden\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 34\u001b[0;31m     \u001b[0mgates\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mw_ih\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mb_ih\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mw_hh\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mb_hh\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     35\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     36\u001b[0m     \u001b[0mingate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mforgetgate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcellgate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutgate\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgates\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchunk\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "\"\"\" Train \"\"\"\n",
    "\n",
    "if config['task'] == 'train':\n",
    "\n",
    "    # save every epoch for visualization\n",
    "    train_loss_record = []\n",
    "    valid_loss_record = []\n",
    "    best_record = 10.0\n",
    "\n",
    "    # training\n",
    "    print('Experiment: {}\\n'.format(config['experiment_name']))\n",
    "\n",
    "    while epoch < config['training']['num_epochs']:\n",
    "\n",
    "        print('Start Epoch {} Training...'.format(epoch))\n",
    "\n",
    "        # loss\n",
    "        train_loss = []\n",
    "        train_loss_sum = []\n",
    "        # dataloader\n",
    "        train_dataloader = DataLoader(dataset=trainDS, shuffle=True, batch_size=1)\n",
    "\n",
    "        for idx, data in enumerate(train_dataloader, 0):\n",
    "\n",
    "            # get data\n",
    "            s1, s2, label = data\n",
    "\n",
    "            # clear gradients\n",
    "            optimizer.zero_grad()\n",
    "\n",
    "            # input\n",
    "            output = siamese(s1, s2)\n",
    "            output = output.squeeze(0)\n",
    "\n",
    "            # label cuda\n",
    "            label = Variable(label)\n",
    "            if torch.cuda.is_available():\n",
    "                label = label.cuda()\n",
    "\n",
    "            # loss backward\n",
    "            loss = criterion(output, label)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            train_loss.append(loss.data.cpu())\n",
    "            train_loss_sum.append(loss.data.cpu())\n",
    "\n",
    "            # Every once and a while check on the loss\n",
    "            if ((idx + 1) % 5000) == 0:\n",
    "                print(train_log_string % (datetime.now(), epoch, idx + 1, len(train), np.mean(train_loss)))\n",
    "                train_loss = []\n",
    "\n",
    "        # Record at every epoch\n",
    "        print('Train Loss at epoch {}: {}\\n'.format(epoch, np.mean(train_loss_sum)))\n",
    "        train_loss_record.append(np.mean(train_loss_sum))\n",
    "\n",
    "        # Valid\n",
    "        print('Epoch {} Validating...'.format(epoch))\n",
    "\n",
    "        # loss\n",
    "        valid_loss = []\n",
    "        # dataloader\n",
    "        valid_dataloader = DataLoader(dataset=validDS, shuffle=True, batch_size=1)\n",
    "\n",
    "        for idx, data in enumerate(valid_dataloader, 0):\n",
    "            # get data\n",
    "            s1, s2, label = data\n",
    "\n",
    "            # input\n",
    "            output = siamese(s1, s2)\n",
    "            output = output.squeeze(0)\n",
    "\n",
    "            # label cuda\n",
    "            label = Variable(label)\n",
    "            if torch.cuda.is_available():\n",
    "                label = label.cuda()\n",
    "\n",
    "            # loss\n",
    "            loss = criterion(output, label)\n",
    "            valid_loss.append(loss.data.cpu())\n",
    "\n",
    "        print(valid_log_string % (datetime.now(), epoch, np.mean(valid_loss)))\n",
    "        # Record\n",
    "        valid_loss_record.append(np.mean(valid_loss))\n",
    "        epoch += 1\n",
    "\n",
    "        if np.mean(valid_loss)-np.mean(train_loss_sum) > 0.02:\n",
    "             print(\"Early Stopping!\")\n",
    "             break\n",
    "\n",
    "        # Keep track of best record\n",
    "        if np.mean(valid_loss) < best_record:\n",
    "            best_record = np.mean(valid_loss)\n",
    "            # save the best model\n",
    "            state_dict = {\n",
    "                'epoch': epoch,\n",
    "                'siamese': siamese.state_dict(),\n",
    "                'optimizer': optimizer.state_dict(),\n",
    "            }\n",
    "            torch.save(state_dict, ckpt_path)\n",
    "            print('Model saved!\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5000, 1)\n",
      "Inference Done.\n",
      "Result has writtn to res/result_1000.txt , Good Luck!\n"
     ]
    }
   ],
   "source": [
    "testDS = mytestDS(test_data, all_sents)\n",
    "# Do not shuffle here\n",
    "test_dataloader = DataLoader(dataset=testDS, num_workers=2, batch_size=1)\n",
    "\n",
    "result = []\n",
    "for idx, data in enumerate(test_dataloader, 0):\n",
    "\n",
    "    # get data\n",
    "    s1, s2 = data\n",
    "\n",
    "    # input\n",
    "    output = siamese(s1,s2)\n",
    "    output = output.squeeze(0)\n",
    "\n",
    "    # feed output into softmax to get prob prediction\n",
    "    sm = nn.Softmax(dim=1)\n",
    "    res = sm(output.data)[:,1]\n",
    "    result += res.data.tolist()\n",
    "\n",
    "result = pd.DataFrame(result)\n",
    "print(result.shape)\n",
    "print('Inference Done.')\n",
    "res_path = os.path.join(config['result']['filepath'], 'result_1000.txt')\n",
    "result.to_csv(res_path, header=False, index=False)\n",
    "print('Result has writtn to', res_path, ', Good Luck!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}