stemming/1.lstm-seq2seq-beam.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from sklearn.utils import shuffle\n",
    "import re\n",
    "import time\n",
    "import collections\n",
    "import os\n",
    "import itertools\n",
    "from sklearn.cross_validation import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_dataset(words, n_words, atleast=1):\n",
    "    count = [['GO', 0], ['PAD', 1], ['EOS', 2], ['UNK', 3]]\n",
    "    counter = collections.Counter(words).most_common(n_words)\n",
    "    counter = [i for i in counter if i[1] >= atleast]\n",
    "    count.extend(counter)\n",
    "    dictionary = dict()\n",
    "    for word, _ in count:\n",
    "        dictionary[word] = len(dictionary)\n",
    "    data = list()\n",
    "    unk_count = 0\n",
    "    for word in words:\n",
    "        index = dictionary.get(word, 0)\n",
    "        if index == 0:\n",
    "            unk_count += 1\n",
    "        data.append(index)\n",
    "    count[0][1] = unk_count\n",
    "    reversed_dictionary = dict(zip(dictionary.values(), dictionary.keys()))\n",
    "    return data, count, dictionary, reversed_dictionary\n",
    "\n",
    "def add_start_end(string):\n",
    "    string = string.split()\n",
    "    strings = []\n",
    "    for s in string:\n",
    "        s = list(s)\n",
    "        s[0] = '<%s'%(s[0])\n",
    "        s[-1] = '%s>'%(s[-1])\n",
    "        strings.extend(s)\n",
    "    return strings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000 10000\n"
     ]
    }
   ],
   "source": [
    "with open('lemmatization-en.txt','r') as fopen:\n",
    "    texts = fopen.read().split('\\n')\n",
    "after, before = [], []\n",
    "for i in texts[:10000]:\n",
    "    splitted = i.encode('ascii', 'ignore').decode(\"utf-8\").lower().split('\\t')\n",
    "    if len(splitted) < 2:\n",
    "        continue\n",
    "    after.append(add_start_end(splitted[0]))\n",
    "    before.append(add_start_end(splitted[1]))\n",
    "    \n",
    "print(len(after),len(before))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab from size: 65\n",
      "Most common words [('e', 9763), ('i', 7229), ('n', 6177), ('s>', 6095), ('o', 5769), ('a', 5633)]\n",
      "Sample data [46, 5, 11, 14, 35, 47, 4, 6, 10, 39] ['<f', 'i', 'r', 's', 't>', '<t', 'e', 'n', 't', 'h>']\n",
      "filtered vocab size: 69\n",
      "% of vocab used: 106.15%\n"
     ]
    }
   ],
   "source": [
    "concat_from = list(itertools.chain(*before))\n",
    "vocabulary_size_from = len(list(set(concat_from)))\n",
    "data_from, count_from, dictionary_from, rev_dictionary_from = build_dataset(concat_from, vocabulary_size_from)\n",
    "print('vocab from size: %d'%(vocabulary_size_from))\n",
    "print('Most common words', count_from[4:10])\n",
    "print('Sample data', data_from[:10], [rev_dictionary_from[i] for i in data_from[:10]])\n",
    "print('filtered vocab size:',len(dictionary_from))\n",
    "print(\"% of vocab used: {}%\".format(round(len(dictionary_from)/vocabulary_size_from,4)*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab from size: 90\n",
      "Most common words [('o', 5631), ('a', 5471), ('e', 5471), ('i', 4890), ('r', 4252), ('<c', 4134)]\n",
      "Sample data [83, 59, 57, 59, 55, 57, 59, 55, 55, 57] ['<1>', '<1', '0>', '<1', '0', '0>', '<1', '0', '0', '0>']\n",
      "filtered vocab size: 94\n",
      "% of vocab used: 104.44%\n"
     ]
    }
   ],
   "source": [
    "concat_to = list(itertools.chain(*after))\n",
    "vocabulary_size_to = len(list(set(concat_to)))\n",
    "data_to, count_to, dictionary_to, rev_dictionary_to = build_dataset(concat_to, vocabulary_size_to)\n",
    "print('vocab from size: %d'%(vocabulary_size_to))\n",
    "print('Most common words', count_to[4:10])\n",
    "print('Sample data', data_to[:10], [rev_dictionary_to[i] for i in data_to[:10]])\n",
    "print('filtered vocab size:',len(dictionary_to))\n",
    "print(\"% of vocab used: {}%\".format(round(len(dictionary_to)/vocabulary_size_to,4)*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "GO = dictionary_from['GO']\n",
    "PAD = dictionary_from['PAD']\n",
    "EOS = dictionary_from['EOS']\n",
    "UNK = dictionary_from['UNK']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(after)):\n",
    "    after[i].append('EOS')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Stemmer:\n",
    "    def __init__(self, size_layer, num_layers, embedded_size, \n",
    "                 from_dict_size, to_dict_size, learning_rate, \n",
    "                 dropout = 0.5, beam_width = 15):\n",
    "        \n",
    "        def lstm_cell(reuse=False):\n",
    "            return tf.nn.rnn_cell.LSTMCell(size_layer, reuse=reuse)\n",
    "        \n",
    "        self.X = tf.placeholder(tf.int32, [None, None])\n",
    "        self.Y = tf.placeholder(tf.int32, [None, None])\n",
    "        self.X_seq_len = tf.count_nonzero(self.X, 1, dtype=tf.int32)\n",
    "        self.Y_seq_len = tf.count_nonzero(self.Y, 1, dtype=tf.int32)\n",
    "        batch_size = tf.shape(self.X)[0]\n",
    "        # encoder\n",
    "        encoder_embeddings = tf.Variable(tf.random_uniform([from_dict_size, embedded_size], -1, 1))\n",
    "        encoder_embedded = tf.nn.embedding_lookup(encoder_embeddings, self.X)\n",
    "        encoder_cells = tf.nn.rnn_cell.MultiRNNCell([lstm_cell() for _ in range(num_layers)])\n",
    "        encoder_dropout = tf.contrib.rnn.DropoutWrapper(encoder_cells, output_keep_prob = 0.5)\n",
    "        self.encoder_out, self.encoder_state = tf.nn.dynamic_rnn(cell = encoder_dropout, \n",
    "                                                                 inputs = encoder_embedded, \n",
    "                                                                 sequence_length = self.X_seq_len,\n",
    "                                                                 dtype = tf.float32)\n",
    "        \n",
    "        self.encoder_state = tuple(self.encoder_state[-1] for _ in range(num_layers))\n",
    "        main = tf.strided_slice(self.Y, [0, 0], [batch_size, -1], [1, 1])\n",
    "        decoder_input = tf.concat([tf.fill([batch_size, 1], GO), main], 1)\n",
    "        # decoder\n",
    "        decoder_embeddings = tf.Variable(tf.random_uniform([to_dict_size, embedded_size], -1, 1))\n",
    "        decoder_cells = tf.nn.rnn_cell.MultiRNNCell([lstm_cell() for _ in range(num_layers)])\n",
    "        dense_layer = tf.layers.Dense(to_dict_size)\n",
    "        training_helper = tf.contrib.seq2seq.ScheduledEmbeddingTrainingHelper(\n",
    "                inputs = tf.nn.embedding_lookup(decoder_embeddings, decoder_input),\n",
    "                sequence_length = self.Y_seq_len,\n",
    "                embedding = decoder_embeddings,\n",
    "                sampling_probability = 0.5,\n",
    "                time_major = False)\n",
    "        training_decoder = tf.contrib.seq2seq.BasicDecoder(\n",
    "                cell = decoder_cells,\n",
    "                helper = training_helper,\n",
    "                initial_state = self.encoder_state,\n",
    "                output_layer = dense_layer)\n",
    "        training_decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "                decoder = training_decoder,\n",
    "                impute_finished = True,\n",
    "                maximum_iterations = tf.reduce_max(self.Y_seq_len))\n",
    "        predicting_decoder = tf.contrib.seq2seq.BeamSearchDecoder(\n",
    "                cell = decoder_cells,\n",
    "                embedding = decoder_embeddings,\n",
    "                start_tokens = tf.tile(tf.constant([GO], dtype=tf.int32), [batch_size]),\n",
    "                end_token = EOS,\n",
    "                initial_state = tf.contrib.seq2seq.tile_batch(self.encoder_state, beam_width),\n",
    "                beam_width = beam_width,\n",
    "                output_layer = dense_layer,\n",
    "                length_penalty_weight = 0.0)\n",
    "        predicting_decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "                decoder = predicting_decoder,\n",
    "                impute_finished = False,\n",
    "                maximum_iterations = 2 * tf.reduce_max(self.X_seq_len))\n",
    "        self.training_logits = training_decoder_output.rnn_output\n",
    "        self.predicting_ids = predicting_decoder_output.predicted_ids[:, :, 0]\n",
    "        masks = tf.sequence_mask(self.Y_seq_len, tf.reduce_max(self.Y_seq_len), dtype=tf.float32)\n",
    "        self.cost = tf.contrib.seq2seq.sequence_loss(logits = self.training_logits,\n",
    "                                                     targets = self.Y,\n",
    "                                                     weights = masks)\n",
    "        self.optimizer = tf.train.AdamOptimizer(learning_rate).minimize(self.cost)\n",
    "        \n",
    "        y_t = tf.argmax(self.training_logits,axis=2)\n",
    "        y_t = tf.cast(y_t, tf.int32)\n",
    "        self.prediction = tf.boolean_mask(y_t, masks)\n",
    "        mask_label = tf.boolean_mask(self.Y, masks)\n",
    "        correct_pred = tf.equal(self.prediction, mask_label)\n",
    "        correct_index = tf.cast(correct_pred, tf.float32)\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "size_layer = 128\n",
    "num_layers = 2\n",
    "embedded_size = 64\n",
    "learning_rate = 1e-3\n",
    "batch_size = 32\n",
    "epoch = 15"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/gradients_impl.py:112: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Stemmer(size_layer, num_layers, embedded_size, len(dictionary_from), \n",
    "                len(dictionary_to), learning_rate,batch_size)\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def str_idx(corpus, dic):\n",
    "    X = []\n",
    "    for i in corpus:\n",
    "        ints = []\n",
    "        for k in i:\n",
    "            try:\n",
    "                ints.append(dic[k])\n",
    "            except Exception as e:\n",
    "                ints.append(UNK)\n",
    "        X.append(ints)\n",
    "    return X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = str_idx(before, dictionary_from)\n",
    "Y = str_idx(after, dictionary_to)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X, test_X, train_Y, test_Y = train_test_split(X, Y, test_size = 0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pad_sentence_batch(sentence_batch, pad_int):\n",
    "    padded_seqs = []\n",
    "    seq_lens = []\n",
    "    max_sentence_len = max([len(sentence) for sentence in sentence_batch])\n",
    "    for sentence in sentence_batch:\n",
    "        padded_seqs.append(sentence + [pad_int] * (max_sentence_len - len(sentence)))\n",
    "        seq_lens.append(len(sentence))\n",
    "    return padded_seqs, seq_lens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 250/250 [00:09<00:00, 27.40it/s, accuracy=0.66, cost=1.19]  \n",
      "test minibatch loop: 100%|██████████| 63/63 [00:01<00:00, 62.98it/s, accuracy=0.601, cost=1.41]\n",
      "train minibatch loop:   1%|          | 3/250 [00:00<00:09, 25.11it/s, accuracy=0.639, cost=1.27]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, avg loss: 1.756106, avg accuracy: 0.525552\n",
      "epoch: 0, avg loss test: 1.498321, avg accuracy test: 0.581786\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 250/250 [00:08<00:00, 28.40it/s, accuracy=0.689, cost=1.08] \n",
      "test minibatch loop: 100%|██████████| 63/63 [00:00<00:00, 65.49it/s, accuracy=0.71, cost=1.03]  \n",
      "train minibatch loop:   1%|          | 3/250 [00:00<00:08, 28.48it/s, accuracy=0.753, cost=0.88]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 1, avg loss: 1.253468, avg accuracy: 0.640075\n",
      "epoch: 1, avg loss test: 1.151150, avg accuracy test: 0.681588\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 250/250 [00:08<00:00, 27.90it/s, accuracy=0.661, cost=1.23] \n",
      "test minibatch loop: 100%|██████████| 63/63 [00:00<00:00, 66.89it/s, accuracy=0.694, cost=1.13] \n",
      "train minibatch loop:   1%|          | 3/250 [00:00<00:08, 27.59it/s, accuracy=0.724, cost=0.966]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 2, avg loss: 1.010481, avg accuracy: 0.711142\n",
      "epoch: 2, avg loss test: 0.951096, avg accuracy test: 0.739199\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 250/250 [00:08<00:00, 28.55it/s, accuracy=0.794, cost=0.751]\n",
      "test minibatch loop: 100%|██████████| 63/63 [00:00<00:00, 66.33it/s, accuracy=0.76, cost=0.811] \n",
      "train minibatch loop:   1%|          | 3/250 [00:00<00:08, 28.14it/s, accuracy=0.852, cost=0.512]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 3, avg loss: 0.800103, avg accuracy: 0.769817\n",
      "epoch: 3, avg loss test: 0.731772, avg accuracy test: 0.800546\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 250/250 [00:08<00:00, 28.46it/s, accuracy=0.84, cost=0.513] \n",
      "test minibatch loop: 100%|██████████| 63/63 [00:00<00:00, 66.85it/s, accuracy=0.858, cost=0.472]\n",
      "train minibatch loop:   1%|          | 3/250 [00:00<00:08, 29.71it/s, accuracy=0.863, cost=0.447]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 4, avg loss: 0.633038, avg accuracy: 0.817272\n",
      "epoch: 4, avg loss test: 0.583872, avg accuracy test: 0.838429\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 250/250 [00:08<00:00, 28.66it/s, accuracy=0.871, cost=0.431]\n",
      "test minibatch loop: 100%|██████████| 63/63 [00:00<00:00, 66.78it/s, accuracy=0.87, cost=0.444] \n",
      "train minibatch loop:   1%|          | 3/250 [00:00<00:08, 29.53it/s, accuracy=0.856, cost=0.481]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 5, avg loss: 0.494747, avg accuracy: 0.857036\n",
      "epoch: 5, avg loss test: 0.458256, avg accuracy test: 0.877494\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 250/250 [00:08<00:00, 27.54it/s, accuracy=0.89, cost=0.375] \n",
      "test minibatch loop: 100%|██████████| 63/63 [00:00<00:00, 66.87it/s, accuracy=0.915, cost=0.279]\n",
      "train minibatch loop:   1%|          | 3/250 [00:00<00:09, 26.55it/s, accuracy=0.925, cost=0.299]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 6, avg loss: 0.396081, avg accuracy: 0.887341\n",
      "epoch: 6, avg loss test: 0.396311, avg accuracy test: 0.893787\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 250/250 [00:08<00:00, 28.53it/s, accuracy=0.956, cost=0.233]\n",
      "test minibatch loop: 100%|██████████| 63/63 [00:00<00:00, 67.34it/s, accuracy=0.938, cost=0.232]\n",
      "train minibatch loop:   2%|▏         | 4/250 [00:00<00:08, 29.47it/s, accuracy=0.898, cost=0.32] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 7, avg loss: 0.327660, avg accuracy: 0.907406\n",
      "epoch: 7, avg loss test: 0.325242, avg accuracy test: 0.915313\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 250/250 [00:08<00:00, 26.76it/s, accuracy=0.913, cost=0.274]\n",
      "test minibatch loop: 100%|██████████| 63/63 [00:00<00:00, 66.11it/s, accuracy=0.924, cost=0.321]\n",
      "train minibatch loop:   1%|          | 3/250 [00:00<00:09, 25.60it/s, accuracy=0.933, cost=0.236]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 8, avg loss: 0.267190, avg accuracy: 0.925086\n",
      "epoch: 8, avg loss test: 0.276169, avg accuracy test: 0.932801\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 250/250 [00:08<00:00, 28.68it/s, accuracy=0.938, cost=0.231]\n",
      "test minibatch loop: 100%|██████████| 63/63 [00:00<00:00, 66.17it/s, accuracy=0.904, cost=0.298]\n",
      "train minibatch loop:   1%|          | 3/250 [00:00<00:09, 26.02it/s, accuracy=0.925, cost=0.224]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 9, avg loss: 0.227773, avg accuracy: 0.936217\n",
      "epoch: 9, avg loss test: 0.253238, avg accuracy test: 0.936522\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 250/250 [00:08<00:00, 28.67it/s, accuracy=0.949, cost=0.174]\n",
      "test minibatch loop: 100%|██████████| 63/63 [00:00<00:00, 66.87it/s, accuracy=0.943, cost=0.219]\n",
      "train minibatch loop:   1%|          | 3/250 [00:00<00:08, 28.69it/s, accuracy=0.914, cost=0.293]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 10, avg loss: 0.194200, avg accuracy: 0.946426\n",
      "epoch: 10, avg loss test: 0.220840, avg accuracy test: 0.947531\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 250/250 [00:08<00:00, 28.71it/s, accuracy=0.951, cost=0.189] \n",
      "test minibatch loop: 100%|██████████| 63/63 [00:00<00:00, 67.24it/s, accuracy=0.969, cost=0.106] \n",
      "train minibatch loop:   1%|          | 3/250 [00:00<00:09, 26.29it/s, accuracy=0.94, cost=0.269] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 11, avg loss: 0.163477, avg accuracy: 0.955249\n",
      "epoch: 11, avg loss test: 0.205023, avg accuracy test: 0.949461\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 250/250 [00:08<00:00, 28.38it/s, accuracy=0.958, cost=0.143] \n",
      "test minibatch loop: 100%|██████████| 63/63 [00:00<00:00, 62.57it/s, accuracy=0.965, cost=0.16]  \n",
      "train minibatch loop:   1%|          | 3/250 [00:00<00:09, 25.67it/s, accuracy=0.977, cost=0.0846]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 12, avg loss: 0.147347, avg accuracy: 0.959543\n",
      "epoch: 12, avg loss test: 0.191767, avg accuracy test: 0.955275\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 250/250 [00:08<00:00, 28.80it/s, accuracy=0.981, cost=0.085] \n",
      "test minibatch loop: 100%|██████████| 63/63 [00:00<00:00, 66.97it/s, accuracy=0.969, cost=0.129] \n",
      "train minibatch loop:   1%|          | 3/250 [00:00<00:08, 28.02it/s, accuracy=0.969, cost=0.137] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 13, avg loss: 0.124258, avg accuracy: 0.966085\n",
      "epoch: 13, avg loss test: 0.168982, avg accuracy test: 0.961098\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 250/250 [00:08<00:00, 28.69it/s, accuracy=0.976, cost=0.0821]\n",
      "test minibatch loop: 100%|██████████| 63/63 [00:00<00:00, 67.08it/s, accuracy=0.955, cost=0.135] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 14, avg loss: 0.110231, avg accuracy: 0.970329\n",
      "epoch: 14, avg loss test: 0.152320, avg accuracy test: 0.966352\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "from sklearn.utils import shuffle\n",
    "import time\n",
    "\n",
    "for EPOCH in range(epoch):\n",
    "    lasttime = time.time()\n",
    "    total_loss, total_accuracy, total_loss_test, total_accuracy_test = 0, 0, 0, 0\n",
    "    train_X, train_Y = shuffle(train_X, train_Y)\n",
    "    test_X, test_Y = shuffle(test_X, test_Y)\n",
    "    pbar = tqdm(range(0, len(train_X), batch_size), desc='train minibatch loop')\n",
    "    for k in pbar:\n",
    "        index = min(k+batch_size,len(train_X))\n",
    "        batch_x, seq_x = pad_sentence_batch(train_X[k: k+batch_size], PAD)\n",
    "        batch_y, seq_y = pad_sentence_batch(train_Y[k: k+batch_size], PAD)\n",
    "        acc, loss, _ = sess.run([model.accuracy, model.cost, model.optimizer], \n",
    "                                      feed_dict={model.X:batch_x,\n",
    "                                                model.Y:batch_y})\n",
    "        total_loss += loss\n",
    "        total_accuracy += acc\n",
    "        pbar.set_postfix(cost=loss, accuracy = acc)\n",
    "        \n",
    "    pbar = tqdm(range(0, len(test_X), batch_size), desc='test minibatch loop')\n",
    "    for k in pbar:\n",
    "        index = min(k+batch_size,len(test_X))\n",
    "        batch_x, seq_x = pad_sentence_batch(test_X[k: k+batch_size], PAD)\n",
    "        batch_y, seq_y = pad_sentence_batch(test_Y[k: k+batch_size], PAD)\n",
    "        acc, loss = sess.run([model.accuracy, model.cost], \n",
    "                                      feed_dict={model.X:batch_x,\n",
    "                                                model.Y:batch_y})\n",
    "        total_loss_test += loss\n",
    "        total_accuracy_test += acc\n",
    "        pbar.set_postfix(cost=loss, accuracy = acc)\n",
    "        \n",
    "    total_loss /= (len(train_X) / batch_size)\n",
    "    total_accuracy /= (len(train_X) / batch_size)\n",
    "    total_loss_test /= (len(test_X) / batch_size)\n",
    "    total_accuracy_test /= (len(test_X) / batch_size)\n",
    "        \n",
    "    print('epoch: %d, avg loss: %f, avg accuracy: %f'%(EPOCH, total_loss, total_accuracy))\n",
    "    print('epoch: %d, avg loss test: %f, avg accuracy test: %f'%(EPOCH, total_loss_test, total_accuracy_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "predicted = sess.run(model.predicting_ids, \n",
    "                     feed_dict={model.X:batch_x})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "row 1\n",
      "BEFORE: <abetting>\n",
      "REAL AFTER: <abet>\n",
      "PREDICTED AFTER: <abett> \n",
      "\n",
      "row 2\n",
      "BEFORE: <bruising>\n",
      "REAL AFTER: <bruise>\n",
      "PREDICTED AFTER: <bruise> \n",
      "\n",
      "row 3\n",
      "BEFORE: <crash-lands>\n",
      "REAL AFTER: <crash-land>\n",
      "PREDICTED AFTER: <crash-land> \n",
      "\n",
      "row 4\n",
      "BEFORE: <calenders>\n",
      "REAL AFTER: <calender>\n",
      "PREDICTED AFTER: <calender> \n",
      "\n",
      "row 5\n",
      "BEFORE: <clacking>\n",
      "REAL AFTER: <clack>\n",
      "PREDICTED AFTER: <clack> \n",
      "\n",
      "row 6\n",
      "BEFORE: <cooked>\n",
      "REAL AFTER: <cook>\n",
      "PREDICTED AFTER: <cook> \n",
      "\n",
      "row 7\n",
      "BEFORE: <corrupted>\n",
      "REAL AFTER: <corrupt>\n",
      "PREDICTED AFTER: <corrupt> \n",
      "\n",
      "row 8\n",
      "BEFORE: <coronas>\n",
      "REAL AFTER: <corona>\n",
      "PREDICTED AFTER: <corona> \n",
      "\n",
      "row 9\n",
      "BEFORE: <bettings>\n",
      "REAL AFTER: <betting>\n",
      "PREDICTED AFTER: <betting> \n",
      "\n",
      "row 10\n",
      "BEFORE: <channelling>\n",
      "REAL AFTER: <channel>\n",
      "PREDICTED AFTER: <channell> \n",
      "\n",
      "row 11\n",
      "BEFORE: <clasps>\n",
      "REAL AFTER: <clasp>\n",
      "PREDICTED AFTER: <clasp> \n",
      "\n",
      "row 12\n",
      "BEFORE: <carousing>\n",
      "REAL AFTER: <carouse>\n",
      "PREDICTED AFTER: <carouse> \n",
      "\n",
      "row 13\n",
      "BEFORE: <bluestones>\n",
      "REAL AFTER: <bluestone>\n",
      "PREDICTED AFTER: <bluestone> \n",
      "\n",
      "row 14\n",
      "BEFORE: <creatinines>\n",
      "REAL AFTER: <creatinine>\n",
      "PREDICTED AFTER: <creatinine> \n",
      "\n",
      "row 15\n",
      "BEFORE: <assesses>\n",
      "REAL AFTER: <assess>\n",
      "PREDICTED AFTER: <assess> \n",
      "\n",
      "row 16\n",
      "BEFORE: <centralities>\n",
      "REAL AFTER: <centrality>\n",
      "PREDICTED AFTER: <centrality> \n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(batch_x)):\n",
    "    print('row %d'%(i+1))\n",
    "    print('BEFORE:',''.join([rev_dictionary_from[n] for n in batch_x[i] if n not in [0,1,2,3]]))\n",
    "    print('REAL AFTER:',''.join([rev_dictionary_to[n] for n in batch_y[i] if n not in[0,1,2,3]]))\n",
    "    print('PREDICTED AFTER:',''.join([rev_dictionary_to[n] for n in predicted[i] if n not in[0,1,2,3]]),'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}