chatbot/45.attention-is-all-you-need.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from sklearn.utils import shuffle\n",
    "import re\n",
    "import time\n",
    "import collections\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_dataset(words, n_words, atleast=1):\n",
    "    count = [['PAD', 0], ['GO', 1], ['EOS', 2], ['UNK', 3]]\n",
    "    counter = collections.Counter(words).most_common(n_words)\n",
    "    counter = [i for i in counter if i[1] >= atleast]\n",
    "    count.extend(counter)\n",
    "    dictionary = dict()\n",
    "    for word, _ in count:\n",
    "        dictionary[word] = len(dictionary)\n",
    "    data = list()\n",
    "    unk_count = 0\n",
    "    for word in words:\n",
    "        index = dictionary.get(word, 0)\n",
    "        if index == 0:\n",
    "            unk_count += 1\n",
    "        data.append(index)\n",
    "    count[0][1] = unk_count\n",
    "    reversed_dictionary = dict(zip(dictionary.values(), dictionary.keys()))\n",
    "    return data, count, dictionary, reversed_dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "lines = open('movie_lines.txt', encoding='utf-8', errors='ignore').read().split('\\n')\n",
    "conv_lines = open('movie_conversations.txt', encoding='utf-8', errors='ignore').read().split('\\n')\n",
    "\n",
    "id2line = {}\n",
    "for line in lines:\n",
    "    _line = line.split(' +++$+++ ')\n",
    "    if len(_line) == 5:\n",
    "        id2line[_line[0]] = _line[4]\n",
    "        \n",
    "convs = [ ]\n",
    "for line in conv_lines[:-1]:\n",
    "    _line = line.split(' +++$+++ ')[-1][1:-1].replace(\"'\",\"\").replace(\" \",\"\")\n",
    "    convs.append(_line.split(','))\n",
    "    \n",
    "questions = []\n",
    "answers = []\n",
    "\n",
    "for conv in convs:\n",
    "    for i in range(len(conv)-1):\n",
    "        questions.append(id2line[conv[i]])\n",
    "        answers.append(id2line[conv[i+1]])\n",
    "        \n",
    "def clean_text(text):\n",
    "    text = text.lower()\n",
    "    text = re.sub(r\"i'm\", \"i am\", text)\n",
    "    text = re.sub(r\"he's\", \"he is\", text)\n",
    "    text = re.sub(r\"she's\", \"she is\", text)\n",
    "    text = re.sub(r\"it's\", \"it is\", text)\n",
    "    text = re.sub(r\"that's\", \"that is\", text)\n",
    "    text = re.sub(r\"what's\", \"that is\", text)\n",
    "    text = re.sub(r\"where's\", \"where is\", text)\n",
    "    text = re.sub(r\"how's\", \"how is\", text)\n",
    "    text = re.sub(r\"\\'ll\", \" will\", text)\n",
    "    text = re.sub(r\"\\'ve\", \" have\", text)\n",
    "    text = re.sub(r\"\\'re\", \" are\", text)\n",
    "    text = re.sub(r\"\\'d\", \" would\", text)\n",
    "    text = re.sub(r\"\\'re\", \" are\", text)\n",
    "    text = re.sub(r\"won't\", \"will not\", text)\n",
    "    text = re.sub(r\"can't\", \"cannot\", text)\n",
    "    text = re.sub(r\"n't\", \" not\", text)\n",
    "    text = re.sub(r\"n'\", \"ng\", text)\n",
    "    text = re.sub(r\"'bout\", \"about\", text)\n",
    "    text = re.sub(r\"'til\", \"until\", text)\n",
    "    text = re.sub(r\"[-()\\\"#/@;:<>{}`+=~|.!?,]\", \"\", text)\n",
    "    return ' '.join([i.strip() for i in filter(None, text.split())])\n",
    "\n",
    "clean_questions = []\n",
    "for question in questions:\n",
    "    clean_questions.append(clean_text(question))\n",
    "    \n",
    "clean_answers = []    \n",
    "for answer in answers:\n",
    "    clean_answers.append(clean_text(answer))\n",
    "    \n",
    "min_line_length = 2\n",
    "max_line_length = 5\n",
    "short_questions_temp = []\n",
    "short_answers_temp = []\n",
    "\n",
    "i = 0\n",
    "for question in clean_questions:\n",
    "    if len(question.split()) >= min_line_length and len(question.split()) <= max_line_length:\n",
    "        short_questions_temp.append(question)\n",
    "        short_answers_temp.append(clean_answers[i])\n",
    "    i += 1\n",
    "\n",
    "short_questions = []\n",
    "short_answers = []\n",
    "\n",
    "i = 0\n",
    "for answer in short_answers_temp:\n",
    "    if len(answer.split()) >= min_line_length and len(answer.split()) <= max_line_length:\n",
    "        short_answers.append(answer)\n",
    "        short_questions.append(short_questions_temp[i])\n",
    "    i += 1\n",
    "    \n",
    "question_test = short_questions[500:550]\n",
    "answer_test = short_answers[500:550]\n",
    "short_questions = short_questions[:500]\n",
    "short_answers = short_answers[:500]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab from size: 657\n",
      "Most common words [('you', 132), ('is', 78), ('i', 68), ('what', 51), ('it', 50), ('that', 49)]\n",
      "Sample data [7, 28, 129, 35, 61, 42, 12, 22, 82, 225] ['what', 'good', 'stuff', 'she', 'okay', 'they', 'do', 'to', 'hey', 'sweet']\n",
      "filtered vocab size: 661\n",
      "% of vocab used: 100.61%\n"
     ]
    }
   ],
   "source": [
    "concat_from = ' '.join(short_questions+question_test).split()\n",
    "vocabulary_size_from = len(list(set(concat_from)))\n",
    "data_from, count_from, dictionary_from, rev_dictionary_from = build_dataset(concat_from, vocabulary_size_from)\n",
    "print('vocab from size: %d'%(vocabulary_size_from))\n",
    "print('Most common words', count_from[4:10])\n",
    "print('Sample data', data_from[:10], [rev_dictionary_from[i] for i in data_from[:10]])\n",
    "print('filtered vocab size:',len(dictionary_from))\n",
    "print(\"% of vocab used: {}%\".format(round(len(dictionary_from)/vocabulary_size_from,4)*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab from size: 660\n",
      "Most common words [('i', 97), ('you', 91), ('is', 62), ('it', 58), ('not', 47), ('what', 39)]\n",
      "Sample data [12, 216, 5, 4, 94, 25, 59, 10, 8, 79] ['the', 'real', 'you', 'i', 'hope', 'so', 'they', 'do', 'not', 'hi']\n",
      "filtered vocab size: 664\n",
      "% of vocab used: 100.61%\n"
     ]
    }
   ],
   "source": [
    "concat_to = ' '.join(short_answers+answer_test).split()\n",
    "vocabulary_size_to = len(list(set(concat_to)))\n",
    "data_to, count_to, dictionary_to, rev_dictionary_to = build_dataset(concat_to, vocabulary_size_to)\n",
    "print('vocab from size: %d'%(vocabulary_size_to))\n",
    "print('Most common words', count_to[4:10])\n",
    "print('Sample data', data_to[:10], [rev_dictionary_to[i] for i in data_to[:10]])\n",
    "print('filtered vocab size:',len(dictionary_to))\n",
    "print(\"% of vocab used: {}%\".format(round(len(dictionary_to)/vocabulary_size_to,4)*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "GO = dictionary_from['GO']\n",
    "PAD = dictionary_from['PAD']\n",
    "EOS = dictionary_from['EOS']\n",
    "UNK = dictionary_from['UNK']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(short_answers)):\n",
    "    short_answers[i] += ' EOS'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def layer_norm(inputs, epsilon=1e-8):\n",
    "    mean, variance = tf.nn.moments(inputs, [-1], keep_dims=True)\n",
    "    normalized = (inputs - mean) / (tf.sqrt(variance + epsilon))\n",
    "\n",
    "    params_shape = inputs.get_shape()[-1:]\n",
    "    gamma = tf.get_variable('gamma', params_shape, tf.float32, tf.ones_initializer())\n",
    "    beta = tf.get_variable('beta', params_shape, tf.float32, tf.zeros_initializer())\n",
    "    \n",
    "    outputs = gamma * normalized + beta\n",
    "    return outputs\n",
    "\n",
    "\n",
    "def multihead_attn(queries, keys, q_masks, k_masks, future_binding, num_units, num_heads):\n",
    "    \n",
    "    T_q = tf.shape(queries)[1]                                      \n",
    "    T_k = tf.shape(keys)[1]                  \n",
    "\n",
    "    Q = tf.layers.dense(queries, num_units, name='Q')                              \n",
    "    K_V = tf.layers.dense(keys, 2*num_units, name='K_V')    \n",
    "    K, V = tf.split(K_V, 2, -1)        \n",
    "\n",
    "    Q_ = tf.concat(tf.split(Q, num_heads, axis=2), axis=0)                         \n",
    "    K_ = tf.concat(tf.split(K, num_heads, axis=2), axis=0)                    \n",
    "    V_ = tf.concat(tf.split(V, num_heads, axis=2), axis=0)                      \n",
    "\n",
    "    align = tf.matmul(Q_, tf.transpose(K_, [0,2,1]))                      \n",
    "    align = align / np.sqrt(K_.get_shape().as_list()[-1])                 \n",
    "\n",
    "    paddings = tf.fill(tf.shape(align), float('-inf'))                   \n",
    "\n",
    "    key_masks = k_masks                                                 \n",
    "    key_masks = tf.tile(key_masks, [num_heads, 1])                       \n",
    "    key_masks = tf.tile(tf.expand_dims(key_masks, 1), [1, T_q, 1])            \n",
    "    align = tf.where(tf.equal(key_masks, 0), paddings, align)       \n",
    "\n",
    "    if future_binding:\n",
    "        lower_tri = tf.ones([T_q, T_k])                                          \n",
    "        lower_tri = tf.linalg.LinearOperatorLowerTriangular(lower_tri).to_dense()  \n",
    "        masks = tf.tile(tf.expand_dims(lower_tri,0), [tf.shape(align)[0], 1, 1]) \n",
    "        align = tf.where(tf.equal(masks, 0), paddings, align)                      \n",
    "    \n",
    "    align = tf.nn.softmax(align)                                            \n",
    "    query_masks = tf.to_float(q_masks)                                             \n",
    "    query_masks = tf.tile(query_masks, [num_heads, 1])                             \n",
    "    query_masks = tf.tile(tf.expand_dims(query_masks, -1), [1, 1, T_k])            \n",
    "    align *= query_masks                                                           \n",
    "          \n",
    "    outputs = tf.matmul(align, V_)                                                 \n",
    "    outputs = tf.concat(tf.split(outputs, num_heads, axis=0), axis=2)             \n",
    "    outputs += queries                                                             \n",
    "    outputs = layer_norm(outputs)                                                 \n",
    "    return outputs\n",
    "\n",
    "\n",
    "def pointwise_feedforward(inputs, hidden_units, activation=None):\n",
    "    outputs = tf.layers.dense(inputs, 4*hidden_units, activation=activation)\n",
    "    outputs = tf.layers.dense(outputs, hidden_units, activation=None)\n",
    "    outputs += inputs\n",
    "    outputs = layer_norm(outputs)\n",
    "    return outputs\n",
    "\n",
    "\n",
    "def learned_position_encoding(inputs, mask, embed_dim):\n",
    "    T = tf.shape(inputs)[1]\n",
    "    outputs = tf.range(tf.shape(inputs)[1])                # (T_q)\n",
    "    outputs = tf.expand_dims(outputs, 0)                   # (1, T_q)\n",
    "    outputs = tf.tile(outputs, [tf.shape(inputs)[0], 1])   # (N, T_q)\n",
    "    outputs = embed_seq(outputs, T, embed_dim, zero_pad=False, scale=False)\n",
    "    return tf.expand_dims(tf.to_float(mask), -1) * outputs\n",
    "\n",
    "\n",
    "def sinusoidal_position_encoding(inputs, mask, repr_dim):\n",
    "    T = tf.shape(inputs)[1]\n",
    "    pos = tf.reshape(tf.range(0.0, tf.to_float(T), dtype=tf.float32), [-1, 1])\n",
    "    i = np.arange(0, repr_dim, 2, np.float32)\n",
    "    denom = np.reshape(np.power(10000.0, i / repr_dim), [1, -1])\n",
    "    enc = tf.expand_dims(tf.concat([tf.sin(pos / denom), tf.cos(pos / denom)], 1), 0)\n",
    "    return tf.tile(enc, [tf.shape(inputs)[0], 1, 1]) * tf.expand_dims(tf.to_float(mask), -1)\n",
    "\n",
    "\n",
    "def label_smoothing(inputs, epsilon=0.1):\n",
    "    C = inputs.get_shape().as_list()[-1]\n",
    "    return ((1 - epsilon) * inputs) + (epsilon / C)\n",
    "\n",
    "\n",
    "class Chatbot:\n",
    "    def __init__(self, size_layer, embedded_size, from_dict_size, to_dict_size, learning_rate,\n",
    "                 num_blocks = 2,\n",
    "                 num_heads = 8,\n",
    "                 min_freq = 50):\n",
    "        self.X = tf.placeholder(tf.int32,[None,None])\n",
    "        self.Y = tf.placeholder(tf.int32,[None,None])\n",
    "        \n",
    "        self.X_seq_len = tf.count_nonzero(self.X, 1, dtype=tf.int32)\n",
    "        self.Y_seq_len = tf.count_nonzero(self.Y, 1, dtype=tf.int32)\n",
    "        batch_size = tf.shape(self.X)[0]\n",
    "        \n",
    "        encoder_embedding = tf.Variable(tf.random_uniform([from_dict_size, embedded_size], -1, 1))\n",
    "        decoder_embedding = tf.Variable(tf.random_uniform([to_dict_size, embedded_size], -1, 1))\n",
    "        \n",
    "        main = tf.strided_slice(self.Y, [0, 0], [batch_size, -1], [1, 1])\n",
    "        decoder_input = tf.concat([tf.fill([batch_size, 1], GO), main], 1)\n",
    "        \n",
    "        def forward(x, y):\n",
    "            encoder_embedded = tf.nn.embedding_lookup(encoder_embedding, x)\n",
    "            en_masks = tf.sign(x)\n",
    "            encoder_embedded += sinusoidal_position_encoding(x, en_masks, embedded_size)\n",
    "        \n",
    "            for i in range(num_blocks):\n",
    "                with tf.variable_scope('encoder_self_attn_%d'%i,reuse=tf.AUTO_REUSE):\n",
    "                    encoder_embedded = multihead_attn(queries = encoder_embedded,\n",
    "                                             keys = encoder_embedded,\n",
    "                                             q_masks = en_masks,\n",
    "                                             k_masks = en_masks,\n",
    "                                             future_binding = False,\n",
    "                                             num_units = size_layer,\n",
    "                                             num_heads = num_heads)\n",
    "\n",
    "                with tf.variable_scope('encoder_feedforward_%d'%i,reuse=tf.AUTO_REUSE):\n",
    "                    encoder_embedded = pointwise_feedforward(encoder_embedded,\n",
    "                                                    embedded_size,\n",
    "                                                    activation = tf.nn.relu)\n",
    "            \n",
    "            decoder_embedded = tf.nn.embedding_lookup(decoder_embedding, y)\n",
    "            de_masks = tf.sign(y)\n",
    "            decoder_embedded += sinusoidal_position_encoding(y, de_masks, embedded_size)\n",
    "            \n",
    "            for i in range(num_blocks):\n",
    "                with tf.variable_scope('decoder_self_attn_%d'%i,reuse=tf.AUTO_REUSE):\n",
    "                    decoder_embedded = multihead_attn(queries = decoder_embedded,\n",
    "                                         keys = decoder_embedded,\n",
    "                                         q_masks = de_masks,\n",
    "                                         k_masks = de_masks,\n",
    "                                         future_binding = True,\n",
    "                                         num_units = size_layer,\n",
    "                                         num_heads = num_heads)\n",
    "                \n",
    "                with tf.variable_scope('decoder_attn_%d'%i,reuse=tf.AUTO_REUSE):\n",
    "                    decoder_embedded = multihead_attn(queries = decoder_embedded,\n",
    "                                         keys = encoder_embedded,\n",
    "                                         q_masks = de_masks,\n",
    "                                         k_masks = en_masks,\n",
    "                                         future_binding = False,\n",
    "                                         num_units = size_layer,\n",
    "                                         num_heads = num_heads)\n",
    "                \n",
    "                with tf.variable_scope('decoder_feedforward_%d'%i,reuse=tf.AUTO_REUSE):\n",
    "                    decoder_embedded = pointwise_feedforward(decoder_embedded,\n",
    "                                                    embedded_size,\n",
    "                                            activation = tf.nn.relu)\n",
    "            \n",
    "            return tf.layers.dense(decoder_embedded, to_dict_size, reuse=tf.AUTO_REUSE)\n",
    "        \n",
    "        self.training_logits = forward(self.X, decoder_input)\n",
    "        \n",
    "        def cond(i, y, temp):\n",
    "            return i < 2 * tf.reduce_max(self.X_seq_len)\n",
    "        \n",
    "        def body(i, y, temp):\n",
    "            logits = forward(self.X, y)\n",
    "            ids = tf.argmax(logits, -1)[:, i]\n",
    "            ids = tf.expand_dims(ids, -1)\n",
    "            temp = tf.concat([temp[:, 1:], ids], -1)\n",
    "            y = tf.concat([temp[:, -(i+1):], temp[:, :-(i+1)]], -1)\n",
    "            y = tf.reshape(y, [tf.shape(temp)[0], 2 * tf.reduce_max(self.X_seq_len)])\n",
    "            i += 1\n",
    "            return i, y, temp\n",
    "        \n",
    "        target = tf.fill([batch_size, 2 * tf.reduce_max(self.X_seq_len)], GO)\n",
    "        target = tf.cast(target, tf.int64)\n",
    "        self.target = target\n",
    "        \n",
    "        _, self.predicting_ids, _ = tf.while_loop(cond, body, \n",
    "                                                  [tf.constant(0), target, target])\n",
    "        \n",
    "        masks = tf.sequence_mask(self.Y_seq_len, tf.reduce_max(self.Y_seq_len), dtype=tf.float32)\n",
    "        self.cost = tf.contrib.seq2seq.sequence_loss(logits = self.training_logits,\n",
    "                                                     targets = self.Y,\n",
    "                                                     weights = masks)\n",
    "        self.optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate).minimize(self.cost)\n",
    "        y_t = tf.argmax(self.training_logits,axis=2)\n",
    "        y_t = tf.cast(y_t, tf.int32)\n",
    "        self.prediction = tf.boolean_mask(y_t, masks)\n",
    "        mask_label = tf.boolean_mask(self.Y, masks)\n",
    "        correct_pred = tf.equal(self.prediction, mask_label)\n",
    "        correct_index = tf.cast(correct_pred, tf.float32)\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedded_size = 256\n",
    "learning_rate = 0.001\n",
    "batch_size = 16\n",
    "epoch = 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Chatbot(embedded_size, embedded_size, len(dictionary_from), \n",
    "                len(dictionary_to), learning_rate)\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def str_idx(corpus, dic):\n",
    "    X = []\n",
    "    for i in corpus:\n",
    "        ints = []\n",
    "        for k in i.split():\n",
    "            ints.append(dic.get(k,UNK))\n",
    "        X.append(ints)\n",
    "    return X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = str_idx(short_questions, dictionary_from)\n",
    "Y = str_idx(short_answers, dictionary_to)\n",
    "X_test = str_idx(question_test, dictionary_from)\n",
    "Y_test = str_idx(answer_test, dictionary_from)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pad_sentence_batch(sentence_batch, pad_int):\n",
    "    padded_seqs = []\n",
    "    seq_lens = []\n",
    "    max_sentence_len = max([len(sentence) for sentence in sentence_batch])\n",
    "    for sentence in sentence_batch:\n",
    "        padded_seqs.append(sentence + [pad_int] * (max_sentence_len - len(sentence)))\n",
    "        seq_lens.append(len(sentence))\n",
    "    return padded_seqs, seq_lens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[4 2 2 2 2 2 2 2]\n",
      " [4 2 2 2 2 2 2 2]\n",
      " [4 2 2 2 2 2 2 2]\n",
      " [4 2 2 2 2 2 2 2]]\n",
      "epoch: 1, avg loss: 5.366285, avg accuracy: 0.227216\n",
      "\n",
      "[[4 4 4 2 2 2 2 2]\n",
      " [4 4 4 2 2 2 2 2]\n",
      " [4 4 4 2 2 2 2 2]\n",
      " [4 4 4 2 2 2 2 2]]\n",
      "epoch: 2, avg loss: 4.871039, avg accuracy: 0.265193\n",
      "\n",
      "[[ 4 15 15 15 15  2  2  2]\n",
      " [ 4 15 15 15 15  2  2  2]\n",
      " [ 4 15 15 15 15  2  2  2]\n",
      " [ 4 15 15 15 15  2  2  2]]\n",
      "epoch: 3, avg loss: 4.585983, avg accuracy: 0.286159\n",
      "\n",
      "[[ 4 10 10  8 15  2  2  2]\n",
      " [ 4 10 10  8 15  2  2  2]\n",
      " [ 4 10 10  8 15  2  2  2]\n",
      " [ 4 10 10  8 15  2  2  2]]\n",
      "epoch: 4, avg loss: 4.208663, avg accuracy: 0.299763\n",
      "\n",
      "[[ 4 10 25 25  5  7  2  2]\n",
      " [ 4 10 25 25  5  7  2  2]\n",
      " [ 4 10 25 25  5  7  2  2]\n",
      " [ 4 10 25 25  5  7  2  2]]\n",
      "epoch: 5, avg loss: 3.852230, avg accuracy: 0.325830\n",
      "\n",
      "[[ 4 10  8  5  8  5  5  4]\n",
      " [ 4 10  8  5  8  5  5  4]\n",
      " [ 4 10  8  5  8  5  5  4]\n",
      " [ 4 10  8  5  8  5  5  4]]\n",
      "epoch: 6, avg loss: 3.663823, avg accuracy: 0.332594\n",
      "\n",
      "[[ 4 56 25 25  2  2  2  2]\n",
      " [ 4 56 25 25  2  2  2  2]\n",
      " [ 4 56 25 25  2  2  2  2]\n",
      " [ 4 56 25 25  2  2  2  2]]\n",
      "epoch: 7, avg loss: 3.417376, avg accuracy: 0.364567\n",
      "\n",
      "[[ 4 10  8 10  8  5  2  2]\n",
      " [ 4 10  8 10  8  5  2  2]\n",
      " [ 4 10  8 10  8  5  2  2]\n",
      " [ 4 10  8 10  8  5  2  2]]\n",
      "epoch: 8, avg loss: 3.006225, avg accuracy: 0.411157\n",
      "\n",
      "[[ 4 10  8 10  8  2  2  2]\n",
      " [ 4 10  8 10  8  2  2  2]\n",
      " [ 4 56 25 25  2  2  2  2]\n",
      " [ 4 56 25 25  2  2  2  2]]\n",
      "epoch: 9, avg loss: 2.860557, avg accuracy: 0.424249\n",
      "\n",
      "[[  4  25  25  25 600   2   2   2]\n",
      " [  4  25  25  25   4   2   2   2]\n",
      " [  4  56  25  25 600   2   2   2]\n",
      " [  4  56  25  25 600   2   2   2]]\n",
      "epoch: 10, avg loss: 2.466612, avg accuracy: 0.505813\n",
      "\n",
      "[[  4  25  25 600  16   2   2   2]\n",
      " [  4  10   8   2  16   2   2   2]\n",
      " [  4  56  25  16 212   2   2   2]\n",
      " [  4  56  25  16 212   2   2   2]]\n",
      "epoch: 11, avg loss: 2.171221, avg accuracy: 0.571672\n",
      "\n",
      "[[ 4 10 56 25  2  2  2  2]\n",
      " [ 4 10 56 25  2  2  2  2]\n",
      " [ 4 10 56 25  2  2  2  2]\n",
      " [ 4 10 56 25  2  2  2  2]]\n",
      "epoch: 12, avg loss: 2.005238, avg accuracy: 0.601502\n",
      "\n",
      "[[  4  15  25  25 600   2   2   2]\n",
      " [  4  48   4  25   4   2   2   2]\n",
      " [  4  10  20  20  20  61   2   2]\n",
      " [  4  10  20  20  20  16   2   2]]\n",
      "epoch: 13, avg loss: 1.887670, avg accuracy: 0.626562\n",
      "\n",
      "[[  4  56  25  17  17  17  17   2]\n",
      " [  4  56  25  25 600   2   2   2]\n",
      " [  5  11 591  52 583   2   2   2]\n",
      " [  5  11 591  16  95   2   2   2]]\n",
      "epoch: 14, avg loss: 1.851392, avg accuracy: 0.627848\n",
      "\n",
      "[[  4  56  25  25 205  17   2   2]\n",
      " [  4  56  25 600   2   2   2   2]\n",
      " [  4  46  56  25 583   2   2   2]\n",
      " [  4  46  56  25 583   2   2   2]]\n",
      "epoch: 15, avg loss: 1.709170, avg accuracy: 0.657352\n",
      "\n",
      "[[ 4 56 25 25 17  2  2  2]\n",
      " [ 4 22  4  4  4  4  2  2]\n",
      " [ 9  5 37  5  5  5  5  2]\n",
      " [ 9  5 37  5  5  5  5  2]]\n",
      "epoch: 16, avg loss: 1.621592, avg accuracy: 0.667013\n",
      "\n",
      "[[  9   5  17  17  17  13   2   2]\n",
      " [  4  48  56  25  25   2   2   2]\n",
      " [  7   6  11 615   2   2   2   2]\n",
      " [  9   5  90 108  57   2   2   2]]\n",
      "epoch: 17, avg loss: 1.475722, avg accuracy: 0.703159\n",
      "\n",
      "[[  4  18  44  12   7  57   2   2]\n",
      " [  4  48  56  25 205  17   2   2]\n",
      " [ 90  90 165  63 620 606   2   2]\n",
      " [ 90  90 165  63 620  73   2   2]]\n",
      "epoch: 18, avg loss: 1.442613, avg accuracy: 0.691040\n",
      "\n",
      "[[  9  52  11 112   2   2   2   2]\n",
      " [  4  10  10   2   2   2   2   2]\n",
      " [ 90   6 165 121  73  73   2   2]\n",
      " [ 90   6 165  73  73  73   2   2]]\n",
      "epoch: 19, avg loss: 1.447122, avg accuracy: 0.697504\n",
      "\n",
      "[[  9  49  36  27  27   2   2   2]\n",
      " [  4  10  48   4   2   2   2   2]\n",
      " [108 209   2   2   2   2   2   2]\n",
      " [108 209   2   2   2   2   2   2]]\n",
      "epoch: 20, avg loss: 1.401934, avg accuracy: 0.704549\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i in range(epoch):\n",
    "    total_loss, total_accuracy = 0, 0\n",
    "    for k in range(0, len(short_questions), batch_size):\n",
    "        index = min(k+batch_size, len(short_questions))\n",
    "        batch_x, seq_x = pad_sentence_batch(X[k: index], PAD)\n",
    "        batch_y, seq_y = pad_sentence_batch(Y[k: index], PAD)\n",
    "        predicted, accuracy,loss, _ = sess.run([model.predicting_ids, \n",
    "                                                model.accuracy, model.cost, model.optimizer], \n",
    "                                      feed_dict={model.X:batch_x,\n",
    "                                                model.Y:batch_y})\n",
    "        total_loss += loss\n",
    "        total_accuracy += accuracy\n",
    "    total_loss /= (len(short_questions) / batch_size)\n",
    "    total_accuracy /= (len(short_questions) / batch_size)\n",
    "    print(predicted)\n",
    "    print('epoch: %d, avg loss: %f, avg accuracy: %f\\n'%(i+1, total_loss, total_accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "row 1\n",
      "QUESTION: i am a werewolf\n",
      "REAL ANSWER: a werewolf\n",
      "PREDICTED ANSWER: what dead who then then \n",
      "\n",
      "row 2\n",
      "QUESTION: i was dreaming again\n",
      "REAL ANSWER: i would think so\n",
      "PREDICTED ANSWER: i do would i \n",
      "\n",
      "row 3\n",
      "QUESTION: the kitchen\n",
      "REAL ANSWER: very nice\n",
      "PREDICTED ANSWER: very nice \n",
      "\n",
      "row 4\n",
      "QUESTION: the bedroom\n",
      "REAL ANSWER: there is only one bed\n",
      "PREDICTED ANSWER: very nice \n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(batch_x)):\n",
    "    print('row %d'%(i+1))\n",
    "    print('QUESTION:',' '.join([rev_dictionary_from[n] for n in batch_x[i] if n not in [0,1,2,3]]))\n",
    "    print('REAL ANSWER:',' '.join([rev_dictionary_to[n] for n in batch_y[i] if n not in[0,1,2,3]]))\n",
    "    print('PREDICTED ANSWER:',' '.join([rev_dictionary_to[n] for n in predicted[i] if n not in[0,1,2,3]]),'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "row 1\n",
      "QUESTION: but david\n",
      "REAL ANSWER: is here that\n",
      "PREDICTED ANSWER: i heard voices life hal \n",
      "\n",
      "row 2\n",
      "QUESTION: hopeless it is hopeless\n",
      "REAL ANSWER: tell ballet then back\n",
      "PREDICTED ANSWER: i do you believe it it \n",
      "\n",
      "row 3\n",
      "QUESTION: miss price\n",
      "REAL ANSWER: yes learning\n",
      "PREDICTED ANSWER: very nice \n",
      "\n",
      "row 4\n",
      "QUESTION: mr kessler wake up please\n",
      "REAL ANSWER: is here are\n",
      "PREDICTED ANSWER: i am so life ashamed bed \n",
      "\n",
      "row 5\n",
      "QUESTION: there were witnesses\n",
      "REAL ANSWER: why she out\n",
      "PREDICTED ANSWER: i heard voices life hal \n",
      "\n",
      "row 6\n",
      "QUESTION: what about it\n",
      "REAL ANSWER: not you are\n",
      "PREDICTED ANSWER: very nice \n",
      "\n",
      "row 7\n",
      "QUESTION: go on ask them\n",
      "REAL ANSWER: i just home\n",
      "PREDICTED ANSWER: very nice \n",
      "\n",
      "row 8\n",
      "QUESTION: beware the moon\n",
      "REAL ANSWER: seen hi is he\n",
      "PREDICTED ANSWER: and german get these daughter \n",
      "\n",
      "row 9\n",
      "QUESTION: did you hear that\n",
      "REAL ANSWER: is down what\n",
      "PREDICTED ANSWER: i think so ashamed \n",
      "\n",
      "row 10\n",
      "QUESTION: i heard that\n",
      "REAL ANSWER: it here not\n",
      "PREDICTED ANSWER: what difference does that make \n",
      "\n",
      "row 11\n",
      "QUESTION: the hound of the baskervilles\n",
      "REAL ANSWER: heard\n",
      "PREDICTED ANSWER: i think so life good \n",
      "\n",
      "row 12\n",
      "QUESTION: it is moving\n",
      "REAL ANSWER: not you hear\n",
      "PREDICTED ANSWER: i am at north north you \n",
      "\n",
      "row 13\n",
      "QUESTION: nice doggie good boy\n",
      "REAL ANSWER: bill stupid\n",
      "PREDICTED ANSWER: what dead who dead then \n",
      "\n",
      "row 14\n",
      "QUESTION: it sounds far away\n",
      "REAL ANSWER: that pecos baby seen hi\n",
      "PREDICTED ANSWER: it is a secret \n",
      "\n",
      "row 15\n",
      "QUESTION: debbie klein cried a lot\n",
      "REAL ANSWER: is will srai not\n",
      "PREDICTED ANSWER: i do think so sire that \n",
      "\n",
      "row 16\n",
      "QUESTION: what are you doing here\n",
      "REAL ANSWER: is know look i\n",
      "PREDICTED ANSWER: very nice \n",
      "\n"
     ]
    }
   ],
   "source": [
    "batch_x, seq_x = pad_sentence_batch(X_test[:batch_size], PAD)\n",
    "batch_y, seq_y = pad_sentence_batch(Y_test[:batch_size], PAD)\n",
    "predicted = sess.run(model.predicting_ids, feed_dict={model.X:batch_x})\n",
    "\n",
    "for i in range(len(batch_x)):\n",
    "    print('row %d'%(i+1))\n",
    "    print('QUESTION:',' '.join([rev_dictionary_from[n] for n in batch_x[i] if n not in [0,1,2,3]]))\n",
    "    print('REAL ANSWER:',' '.join([rev_dictionary_to[n] for n in batch_y[i] if n not in[0,1,2,3]]))\n",
    "    print('PREDICTED ANSWER:',' '.join([rev_dictionary_to[n] for n in predicted[i] if n not in[0,1,2,3]]),'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}