topic-generator/3.mta.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.nn.rnn_cell import RNNCell, MultiRNNCell"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "123"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels = os.listdir('news')\n",
    "news = ['news/' + i for i in labels if '.json' in i]\n",
    "labels = [i.replace('.json','') for i in labels]\n",
    "len(news)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import malaya"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = malaya.preprocessing._SocialTokenizer().tokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "accept_tokens = ',-.()\"\\''\n",
    "\n",
    "def is_number_regex(s):\n",
    "    if re.match(\"^\\d+?\\.\\d+?$\", s) is None:\n",
    "        return s.isdigit()\n",
    "    return True\n",
    "\n",
    "def detect_money(word):\n",
    "    if word[:2] == 'rm' and is_number_regex(word[2:]):\n",
    "        return True\n",
    "    else:\n",
    "        return False\n",
    "\n",
    "def preprocessing(string):\n",
    "    tokenized = tokenizer(string)\n",
    "    tokenized = [w.lower() for w in tokenized if len(w) > 1 or w in accept_tokens]\n",
    "    tokenized = ['<NUM>' if is_number_regex(w) else w for w in tokenized]\n",
    "    tokenized = ['<MONEY>' if detect_money(w) else w for w in tokenized]\n",
    "    return tokenized\n",
    "\n",
    "def clean_label(label):\n",
    "    string = re.sub('[^A-Za-z\\- ]+', ' ', label)\n",
    "    return re.sub(r'[ ]+', ' ', string.lower()).strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = [clean_label(label) for label in labels]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.utils import shuffle\n",
    "\n",
    "maxlen = 150\n",
    "maxlen_topic = 10\n",
    "min_len = 20\n",
    "\n",
    "x, y = [], []\n",
    "for no, n in enumerate(news):\n",
    "    with open(n) as fopen: \n",
    "        news_ = json.load(fopen)\n",
    "    for row in news_:\n",
    "        if len(row['text'].split()) > min_len:\n",
    "            p = preprocessing(row['text'])\n",
    "            p = p[:maxlen]\n",
    "            x.append(p)\n",
    "            y.append(labels[no])\n",
    "            \n",
    "x, y = shuffle(x, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(14471, 14471)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(x), len(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import collections\n",
    "\n",
    "def build_dataset(words, n_words, atleast=2):\n",
    "    count = [['PAD', 0], ['GO', 1], ['EOS', 2], ['UNK', 3]]\n",
    "    counter = collections.Counter(words).most_common(n_words)\n",
    "    counter = [i for i in counter if i[1] >= atleast]\n",
    "    count.extend(counter)\n",
    "    dictionary = dict()\n",
    "    for word, _ in count:\n",
    "        dictionary[word] = len(dictionary)\n",
    "    data = list()\n",
    "    unk_count = 0\n",
    "    for word in words:\n",
    "        index = dictionary.get(word, 0)\n",
    "        if index == 0:\n",
    "            unk_count += 1\n",
    "        data.append(index)\n",
    "    count[0][1] = unk_count\n",
    "    reversed_dictionary = dict(zip(dictionary.values(), dictionary.keys()))\n",
    "    return data, count, dictionary, reversed_dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab from size: 63032\n",
      "Most common words [(',', 90498), ('.', 80674), ('yang', 32102), ('-', 29965), ('the', 29732), ('dan', 28171)]\n",
      "Sample data [255, 24344, 458, 8433, 131, 5, 400, 2310, 522, 1142] ['produk', 'albothyl', 'tengah', 'meresahkan', 'masyarakat', '.', 'karena', 'mengandung', 'bahan', 'berbahaya']\n",
      "filtered vocab size: 38621\n",
      "% of vocab used: 61.27%\n"
     ]
    }
   ],
   "source": [
    "import itertools\n",
    "\n",
    "concat = list(itertools.chain(*x)) + ' '.join(labels).split()\n",
    "vocabulary_size = len(list(set(concat)))\n",
    "data, count, dictionary, rev_dictionary = build_dataset(concat, vocabulary_size)\n",
    "print('vocab from size: %d'%(vocabulary_size))\n",
    "print('Most common words', count[4:10])\n",
    "print('Sample data', data[:10], [rev_dictionary[i] for i in data[:10]])\n",
    "print('filtered vocab size:',len(dictionary))\n",
    "print(\"% of vocab used: {}%\".format(round(len(dictionary)/vocabulary_size,4)*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(x)):\n",
    "    x.append('EOS')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "GO = dictionary['GO']\n",
    "PAD = dictionary['PAD']\n",
    "EOS = dictionary['EOS']\n",
    "UNK = dictionary['UNK']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.python.ops import variable_scope as vs\n",
    "from tensorflow.python.ops import array_ops\n",
    "from tensorflow.python.ops import nn_ops\n",
    "from tensorflow.python.ops import math_ops\n",
    "from tensorflow.python.ops import gen_array_ops\n",
    "from tensorflow.python.layers import core as layers_core\n",
    "import tensorflow as tf\n",
    "\n",
    "\n",
    "class MTAWrapper(RNNCell):\n",
    "    def __init__(self, cell, memory, v, uf, query_layer, memory_layer, mask=None, \n",
    "                 max_len=100, attention_size=128, state_is_tuple=True\n",
    "                 ):\n",
    "        if not isinstance(cell, RNNCell):\n",
    "            raise TypeError(\"The parameter cell is not RNNCell.\")\n",
    "        self._cell = cell\n",
    "        self.memory = memory\n",
    "        self._state_is_tuple = state_is_tuple\n",
    "        self.attention_size = attention_size\n",
    "        \n",
    "        self.batch_size = tf.shape(self.memory)[0]\n",
    "        self.num_keywords = tf.shape(self.memory)[1]\n",
    "        self.embedding_size = tf.shape(self.memory)[2]\n",
    "\n",
    "        self.coverage_vector = tf.fill([self.batch_size, self.num_keywords], 1.0)\n",
    "        \n",
    "        if mask is None:\n",
    "            self.seq_len = tf.fill([self.batch_size, 1], 1.0)  # inference\n",
    "        else:\n",
    "            self.seq_len = math_ops.reduce_sum(mask, axis=1, keepdims=True)  # training\n",
    "\n",
    "        self.v = v\n",
    "        self.query_layer = query_layer\n",
    "\n",
    "        self.memory_layer = memory_layer\n",
    "        self.u_f = uf\n",
    "        res1 = tf.sigmoid(\n",
    "            tf.matmul(tf.reshape(self.memory, [self.batch_size, -1]), self.u_f))\n",
    "        self.phi_res = self.seq_len * res1\n",
    "    @property\n",
    "    def state_size(self):\n",
    "        return self._cell.state_size\n",
    "\n",
    "    @property\n",
    "    def output_size(self):\n",
    "        return self._cell.output_size\n",
    "\n",
    "    def __call__(self, inputs, state, scope=None):\n",
    "        c_t, h_t = state\n",
    "        dtype = inputs.dtype\n",
    "\n",
    "        with vs.variable_scope(\"topic_attention\"):\n",
    "            keys = self.memory_layer(self.memory)\n",
    "            processed_query = array_ops.expand_dims(self.query_layer(h_t), 1) \n",
    "            score = self.coverage_vector * math_ops.reduce_sum(self.v * math_ops.tanh(keys + processed_query), [2])\n",
    "            score = nn_ops.softmax(score, axis=1)\n",
    "            score_tile = gen_array_ops.tile(array_ops.expand_dims(score, -1), [1, 1, self.embedding_size],\n",
    "                                            name=\"weight\")\n",
    "            mt = math_ops.reduce_sum(self.memory * score_tile, axis=1)\n",
    "            self.coverage_vector = self.coverage_vector - score / self.phi_res\n",
    "        return self._cell(tf.concat([inputs, mt], axis=1), state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Generator:\n",
    "    def __init__(self, size_layer, num_layers, embedded_size, \n",
    "                 dict_size, learning_rate, beam_width = 5):\n",
    "        \n",
    "        def lstm_cell(size, reuse=False):\n",
    "            return tf.nn.rnn_cell.LSTMCell(size, initializer=tf.orthogonal_initializer(),\n",
    "                                           reuse=reuse)\n",
    "        \n",
    "        self.X = tf.placeholder(tf.int32, [None, None])\n",
    "        self.Y = tf.placeholder(tf.int32, [None, None])\n",
    "        self.X_seq_len = tf.count_nonzero(self.X, 1, dtype=tf.int32)\n",
    "        self.Y_seq_len = tf.count_nonzero(self.Y, 1, dtype=tf.int32)\n",
    "        batch_size = tf.shape(self.X)[0]\n",
    "        \n",
    "        embeddings = tf.Variable(tf.random_uniform([dict_size, embedded_size], -1, 1))\n",
    "        topic_embedded = tf.nn.embedding_lookup(embeddings, self.X)\n",
    "        topic_average = tf.reduce_mean(topic_embedded, axis=1)\n",
    "        main = tf.strided_slice(self.Y, [0, 0], [batch_size, -1], [1, 1])\n",
    "        decoder_input = tf.concat([tf.fill([batch_size, 1], GO), main], 1)\n",
    "        \n",
    "        decoder_cells = lstm_cell(size_layer)\n",
    "        self.decoder_cells = decoder_cells\n",
    "        dense_layer = tf.layers.Dense(dict_size)\n",
    "        \n",
    "        self.encoder_state = decoder_cells.zero_state(batch_size=batch_size,\n",
    "                                                                  dtype=tf.float32)\n",
    "        \n",
    "        self.v = tf.get_variable(\"attention_v\", [size_layer])\n",
    "        self.query_layer = tf.layers.Dense(size_layer)\n",
    "        self.memory_layer = tf.layers.Dense(size_layer)\n",
    "        self.uf = tf.get_variable(\"u_f\", [maxlen_topic * embedded_size, maxlen_topic])\n",
    "        \n",
    "        masks = tf.sequence_mask(self.Y_seq_len, tf.reduce_max(self.Y_seq_len), dtype=tf.float32)\n",
    "        \n",
    "        with tf.variable_scope('decode'):\n",
    "            decoder_cells = MTAWrapper(self.decoder_cells, topic_embedded,\n",
    "                                       self.v, self.uf, self.query_layer, self.memory_layer, mask=masks,\n",
    "                                      max_len = maxlen)\n",
    "\n",
    "            training_helper = tf.contrib.seq2seq.TrainingHelper(\n",
    "                    inputs = tf.nn.embedding_lookup(embeddings, decoder_input),\n",
    "                    sequence_length = self.Y_seq_len,\n",
    "                    time_major = False)\n",
    "\n",
    "            training_decoder = tf.contrib.seq2seq.BasicDecoder(\n",
    "                    cell = decoder_cells,\n",
    "                    helper = training_helper,\n",
    "                    initial_state = self.encoder_state,\n",
    "                    output_layer = dense_layer)\n",
    "            training_decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "                    decoder = training_decoder,\n",
    "                    impute_finished = True,\n",
    "                    maximum_iterations = tf.reduce_max(self.Y_seq_len))\n",
    "        \n",
    "        with tf.variable_scope('decode', reuse=True):\n",
    "            tiled_uf = tf.contrib.seq2seq.tile_batch(self.uf, beam_width)\n",
    "            tiled_topic_embedded = tf.contrib.seq2seq.tile_batch(topic_embedded, beam_width)\n",
    "            decoder_cells = MTAWrapper(self.decoder_cells, tiled_topic_embedded,\n",
    "                                       self.v, tiled_uf, self.query_layer, self.memory_layer,\n",
    "                                      max_len = maxlen)\n",
    "        \n",
    "            predicting_decoder = tf.contrib.seq2seq.BeamSearchDecoder(\n",
    "                    cell = decoder_cells,\n",
    "                    embedding = embeddings,\n",
    "                    start_tokens = tf.tile(tf.constant([GO], dtype=tf.int32), [batch_size]),\n",
    "                    end_token = EOS,\n",
    "                    initial_state = tf.contrib.seq2seq.tile_batch(self.encoder_state, beam_width),\n",
    "                    beam_width = beam_width,\n",
    "                    output_layer = dense_layer,\n",
    "                    length_penalty_weight = 0.0)\n",
    "        \n",
    "            predicting_decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "                    decoder = predicting_decoder,\n",
    "                    impute_finished = False,\n",
    "                    maximum_iterations = maxlen)\n",
    "        \n",
    "        self.training_logits = training_decoder_output.rnn_output\n",
    "        self.predicting_ids = predicting_decoder_output.predicted_ids[:, :, 0]\n",
    "        \n",
    "        self.cost = tf.contrib.seq2seq.sequence_loss(logits = self.training_logits,\n",
    "                                                     targets = self.Y,\n",
    "                                                     weights = masks)\n",
    "        self.optimizer = tf.train.AdamOptimizer(learning_rate).minimize(self.cost)\n",
    "        y_t = tf.argmax(self.training_logits,axis=2)\n",
    "        y_t = tf.cast(y_t, tf.int32)\n",
    "        self.prediction = tf.boolean_mask(y_t, masks)\n",
    "        mask_label = tf.boolean_mask(self.Y, masks)\n",
    "        correct_pred = tf.equal(self.prediction, mask_label)\n",
    "        correct_index = tf.cast(correct_pred, tf.float32)\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "size_layer = 256\n",
    "num_layers = 2\n",
    "embedded_size = 128\n",
    "learning_rate = 0.001\n",
    "batch_size = 32\n",
    "epoch = 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Generator(size_layer, num_layers, embedded_size, len(dictionary), \n",
    "                learning_rate)\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def str_idx(corpus, dic):\n",
    "    X = []\n",
    "    for i in corpus:\n",
    "        ints = []\n",
    "        for k in i:\n",
    "            ints.append(dic.get(k,UNK))\n",
    "        X.append(ints)\n",
    "    return X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = [i.split() for i in y]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = str_idx(y, dictionary)\n",
    "Y = str_idx(x, dictionary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pad_sentence_batch(sentence_batch, pad_int):\n",
    "    padded_seqs = []\n",
    "    seq_lens = []\n",
    "    max_sentence_len = max([len(sentence) for sentence in sentence_batch])\n",
    "    for sentence in sentence_batch:\n",
    "        padded_seqs.append(sentence + [pad_int] * (max_sentence_len - len(sentence)))\n",
    "        seq_lens.append(len(sentence))\n",
    "    return padded_seqs, seq_lens\n",
    "\n",
    "def pad_sentence_batch_static(sentence_batch, pad_int):\n",
    "    padded_seqs = []\n",
    "    seq_lens = []\n",
    "    max_sentence_len = maxlen\n",
    "    for sentence in sentence_batch:\n",
    "        padded_seqs.append(sentence + [pad_int] * (max_sentence_len - len(sentence)))\n",
    "        seq_lens.append(maxlen)\n",
    "    return padded_seqs, seq_lens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[11385,  1941, 17886, 17886, 17886, 17886, 17886, 17886, 17886,\n",
       "        29260, 31328, 12976, 17886, 17886, 12976, 17886,  8681, 27175,\n",
       "        17886, 17886, 17886, 17886, 12976,  8681, 27175, 11988, 24553,\n",
       "        11988, 11988, 24553, 11988, 24553, 11988, 23563, 15066, 36818,\n",
       "        36818, 36818,  6224, 16241, 11988, 11988, 11988, 11988, 11988,\n",
       "         6224, 11988, 11988, 11988,  6579,  6579, 37130, 11988, 26193,\n",
       "        26193, 26193, 37130, 26193, 37130, 37130, 11988, 11988, 11988,\n",
       "         6224, 11988, 11988, 11988,  6224, 11988,  8826, 11988,  8826,\n",
       "        37552, 36818, 36818, 36818, 30803, 30803,  6224, 34199, 36818,\n",
       "        36818, 36818, 36818, 30803,  6224,  6224,  6765,  6765,  6224,\n",
       "         6765,  6765,  6765,  6765,  5278,  1304, 10806, 29149,  9831,\n",
       "         9831,  9831,  9831,  9831,  9831, 30803, 30803, 30803, 30803,\n",
       "         6230, 29149, 30803, 30803,  6230, 29149, 30803, 10806, 29149,\n",
       "        10806, 29149,  9831,  9831,  9831,  9831, 30803, 30803, 30803,\n",
       "        29149, 30803,  6230, 29149, 30803, 30803,  6230, 29149, 30803,\n",
       "        10806, 34871, 29149, 30803, 29149, 30803, 30803,  6230, 29149,\n",
       "        30803, 10806, 34871, 29149, 30803,  6224]], dtype=int32)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_x, _ = pad_sentence_batch(X[:1], PAD)\n",
    "sess.run(model.predicting_ids, feed_dict = {model.X: batch_x})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 453/453 [08:03<00:00,  1.01it/s, accuracy=0.0962, cost=6.73]\n",
      "minibatch loop:   0%|          | 0/453 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 1, avg loss: 7.429072, avg accuracy: 0.059118\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 453/453 [08:03<00:00,  1.01it/s, accuracy=0.135, cost=6.03] \n",
      "minibatch loop:   0%|          | 0/453 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 2, avg loss: 6.443972, avg accuracy: 0.114713\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 453/453 [08:03<00:00,  1.01it/s, accuracy=0.172, cost=5.5] \n",
      "minibatch loop:   0%|          | 0/453 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 3, avg loss: 5.930789, avg accuracy: 0.153914\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 453/453 [08:03<00:00,  1.01it/s, accuracy=0.204, cost=5.1] \n",
      "minibatch loop:   0%|          | 0/453 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 4, avg loss: 5.576925, avg accuracy: 0.179522\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 453/453 [08:03<00:00,  1.01it/s, accuracy=0.218, cost=4.78]\n",
      "minibatch loop:   0%|          | 0/453 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 5, avg loss: 5.312454, avg accuracy: 0.196892\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 453/453 [08:03<00:00,  1.01it/s, accuracy=0.23, cost=4.5]  \n",
      "minibatch loop:   0%|          | 0/453 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 6, avg loss: 5.097266, avg accuracy: 0.210725\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 453/453 [08:03<00:00,  1.01it/s, accuracy=0.25, cost=4.26] \n",
      "minibatch loop:   0%|          | 0/453 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 7, avg loss: 4.914798, avg accuracy: 0.222112\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 453/453 [08:03<00:00,  1.01it/s, accuracy=0.267, cost=4.04]\n",
      "minibatch loop:   0%|          | 0/453 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 8, avg loss: 4.756433, avg accuracy: 0.232037\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 453/453 [08:03<00:00,  1.01it/s, accuracy=0.287, cost=3.83]\n",
      "minibatch loop:   0%|          | 0/453 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 9, avg loss: 4.614790, avg accuracy: 0.241598\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 453/453 [08:03<00:00,  1.01it/s, accuracy=0.314, cost=3.64]\n",
      "minibatch loop:   0%|          | 0/453 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 10, avg loss: 4.488174, avg accuracy: 0.250645\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 453/453 [08:03<00:00,  1.01it/s, accuracy=0.331, cost=3.47]\n",
      "minibatch loop:   0%|          | 0/453 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 11, avg loss: 4.373607, avg accuracy: 0.259206\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 453/453 [08:03<00:00,  1.01it/s, accuracy=0.35, cost=3.31] \n",
      "minibatch loop:   0%|          | 0/453 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 12, avg loss: 4.270588, avg accuracy: 0.267425\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 453/453 [08:03<00:00,  1.01it/s, accuracy=0.377, cost=3.17]\n",
      "minibatch loop:   0%|          | 0/453 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 13, avg loss: 4.175698, avg accuracy: 0.275535\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 453/453 [08:03<00:00,  1.00it/s, accuracy=0.399, cost=3.05]\n",
      "minibatch loop:   0%|          | 0/453 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 14, avg loss: 4.087636, avg accuracy: 0.283630\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 453/453 [08:03<00:00,  1.01it/s, accuracy=0.426, cost=2.93]\n",
      "minibatch loop:   0%|          | 0/453 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 15, avg loss: 4.007741, avg accuracy: 0.291590\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 453/453 [08:03<00:00,  1.01it/s, accuracy=0.441, cost=2.81]\n",
      "minibatch loop:   0%|          | 0/453 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 16, avg loss: 3.930929, avg accuracy: 0.299675\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 453/453 [08:03<00:00,  1.01it/s, accuracy=0.465, cost=2.71]\n",
      "minibatch loop:   0%|          | 0/453 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 17, avg loss: 3.859192, avg accuracy: 0.307621\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 453/453 [08:03<00:00,  1.01it/s, accuracy=0.479, cost=2.62]\n",
      "minibatch loop:   0%|          | 0/453 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 18, avg loss: 3.793177, avg accuracy: 0.315043\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 453/453 [08:03<00:00,  1.01it/s, accuracy=0.494, cost=2.53]\n",
      "minibatch loop:   0%|          | 0/453 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 19, avg loss: 3.731221, avg accuracy: 0.322467\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 453/453 [08:03<00:00,  1.01it/s, accuracy=0.512, cost=2.45]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 20, avg loss: 3.672711, avg accuracy: 0.329646\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "for i in range(epoch):\n",
    "    total_loss, total_accuracy = 0, 0\n",
    "    pbar = tqdm(\n",
    "        range(0, len(X), batch_size), desc = 'minibatch loop')\n",
    "    for k in pbar:\n",
    "        index = min(k+batch_size, len(X))\n",
    "        batch_x, seq_x = pad_sentence_batch(X[k: index], PAD)\n",
    "        batch_y, seq_y = pad_sentence_batch(Y[k: index], PAD)\n",
    "        accuracy,loss, _ = sess.run([model.accuracy, model.cost, model.optimizer], \n",
    "                                      feed_dict={model.X:batch_x,\n",
    "                                                model.Y:batch_y})\n",
    "        total_loss += loss\n",
    "        total_accuracy += accuracy\n",
    "        pbar.set_postfix(cost = loss, accuracy = accuracy)\n",
    "    total_loss /= (len(X) / batch_size)\n",
    "    total_accuracy /= (len(X) / batch_size)\n",
    "    print('epoch: %d, avg loss: %f, avg accuracy: %f'%(i+1, total_loss, total_accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_topic = 'isu najib razak mahathir'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_x, _ = pad_sentence_batch(X[:1], PAD)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[28, 134, 253, 112]]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_topic_idx = str_idx([test_topic.split()], dictionary)\n",
    "batch_test, _ = pad_sentence_batch(test_topic_idx, PAD)\n",
    "batch_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'- ( ubah saiz teks ) kuala lumpur - perdana menteri , datuk seri najib tun razak hari ini , perdana menteri , datuk seri najib tun razak dan perdana menteri , datuk seri najib tun razak dan perdana menteri , datuk seri najib tun razak dan perdana menteri , datuk seri najib tun razak dan perdana menteri , datuk seri najib tun razak dan perdana menteri , datuk seri najib tun razak dan perdana menteri , datuk seri najib tun razak dan perdana menteri , datuk seri najib tun razak dan perdana menteri , datuk seri najib tun razak dan perdana menteri , datuk seri najib tun razak dan perdana menteri , datuk seri najib tun razak dan perdana menteri , datuk seri najib tun razak dan perdana menteri , datuk seri najib tun razak dan perdana menteri , datuk seri najib tun razak dan perdana menteri datuk seri'"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predict_test = sess.run(model.predicting_ids, feed_dict = {model.X: batch_test})[0]\n",
    "' '.join([rev_dictionary[i] for i in predict_test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}