neural-machine-translation/46.google-nmt.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '2'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('dataset-bpe.json') as fopen:\n",
    "    data = json.load(fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X = data['train_X']\n",
    "train_Y = data['train_Y']\n",
    "test_X = data['test_X']\n",
    "test_Y = data['test_Y']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "EOS = 2\n",
    "GO = 1\n",
    "vocab_size = 32000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_Y = [i + [2] for i in train_Y]\n",
    "test_Y = [i + [2] for i in test_Y]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.python.util import nest\n",
    "from tensorflow.python.layers.core import Dense\n",
    "\n",
    "def gnmt_residual_fn(inputs, outputs):\n",
    "    def split_input(inp, out):\n",
    "        out_dim = out.get_shape().as_list()[-1]\n",
    "        inp_dim = inp.get_shape().as_list()[-1]\n",
    "        return tf.split(inp, [out_dim, inp_dim - out_dim], axis=-1)\n",
    "    actual_inputs, _ = nest.map_structure(split_input, inputs, outputs)\n",
    "\n",
    "    def assert_shape_match(inp, out):\n",
    "        inp.get_shape().assert_is_compatible_with(out.get_shape())\n",
    "    nest.assert_same_structure(actual_inputs, outputs)\n",
    "    nest.map_structure(assert_shape_match, actual_inputs, outputs)\n",
    "    return nest.map_structure(lambda inp, out: inp + out, actual_inputs, outputs)\n",
    "\n",
    "class GNMTAttentionMultiCell(tf.nn.rnn_cell.MultiRNNCell):\n",
    "\n",
    "    def __init__(self, attention_cell, cells, use_new_attention=True):\n",
    "        cells = [attention_cell] + cells\n",
    "        self.use_new_attention = use_new_attention\n",
    "        super(GNMTAttentionMultiCell, self).__init__(\n",
    "            cells, state_is_tuple=True)\n",
    "\n",
    "    def __call__(self, inputs, state, scope=None):\n",
    "        \"\"\"Run the cell with bottom layer's attention copied to all upper layers.\"\"\"\n",
    "        if not nest.is_sequence(state):\n",
    "            raise ValueError(\n",
    "                \"Expected state to be a tuple of length %d, but received: %s\"\n",
    "                % (len(self.state_size), state))\n",
    "\n",
    "        with tf.variable_scope(scope or \"multi_rnn_cell\"):\n",
    "            new_states = []\n",
    "\n",
    "            with tf.variable_scope(\"cell_0_attention\"):\n",
    "                attention_cell = self._cells[0]\n",
    "                attention_state = state[0]\n",
    "                cur_inp, new_attention_state = attention_cell(\n",
    "                    inputs, attention_state)\n",
    "                new_states.append(new_attention_state)\n",
    "\n",
    "            for i in range(1, len(self._cells)):\n",
    "                with tf.variable_scope(\"cell_%d\" % i):\n",
    "                    cell = self._cells[i]\n",
    "                    cur_state = state[i]\n",
    "\n",
    "                    if self.use_new_attention:\n",
    "                        cur_inp = tf.concat(\n",
    "                            [cur_inp, new_attention_state.attention], -1)\n",
    "                    else:\n",
    "                        cur_inp = tf.concat(\n",
    "                            [cur_inp, attention_state.attention], -1)\n",
    "\n",
    "                    cur_inp, new_state = cell(cur_inp, cur_state)\n",
    "                    new_states.append(new_state)\n",
    "        return cur_inp, tuple(new_states)\n",
    "\n",
    "class Translator:\n",
    "    def __init__(self, size_layer, num_layers, embedded_size, learning_rate, beam_width = 5):\n",
    "        \n",
    "        def cells(size_layer=size_layer,reuse=False):\n",
    "            return tf.nn.rnn_cell.LSTMCell(size_layer,initializer=tf.orthogonal_initializer(),reuse=reuse)\n",
    "        \n",
    "        self.X = tf.placeholder(tf.int32, [None, None])\n",
    "        self.Y = tf.placeholder(tf.int32, [None, None])\n",
    "        self.X_seq_len = tf.count_nonzero(self.X, 1, dtype=tf.int32)\n",
    "        self.Y_seq_len = tf.count_nonzero(self.Y, 1, dtype=tf.int32)\n",
    "        batch_size = tf.shape(self.X)[0]\n",
    "        \n",
    "        embeddings = tf.Variable(tf.random_uniform([vocab_size, embedded_size], -1, 1))\n",
    "        main = tf.strided_slice(self.Y, [0, 0], [batch_size, -1], [1, 1])\n",
    "        decoder_input = tf.concat([tf.fill([batch_size, 1], GO), main], 1)\n",
    "        decoder_embedded = tf.nn.embedding_lookup(embeddings, decoder_input)\n",
    "        \n",
    "        num_residual_layer = num_layers - 2\n",
    "        num_bi_layer = 1\n",
    "        num_ui_layer = num_layers - num_bi_layer\n",
    "        \n",
    "        encoder_outputs, encoder_state = tf.nn.dynamic_rnn(\n",
    "            cell = tf.nn.rnn_cell.MultiRNNCell([cells() for _ in range(num_layers)]), \n",
    "            inputs = tf.nn.embedding_lookup(embeddings, self.X),\n",
    "            sequence_length = self.X_seq_len,\n",
    "            dtype = tf.float32)\n",
    "        \n",
    "        decoder_cells = []\n",
    "        for n in range(num_layers):\n",
    "            cell = cells(size_layer)\n",
    "            if (n >= num_layers - num_residual_layer):\n",
    "                cell = tf.nn.rnn_cell.ResidualWrapper(cell, residual_fn = gnmt_residual_fn)\n",
    "            decoder_cells.append(cell)\n",
    "        attention_cell = decoder_cells.pop(0)\n",
    "        to_dense = tf.layers.Dense(vocab_size)\n",
    "        \n",
    "        with tf.variable_scope('decode'):\n",
    "            attention_mechanism = tf.contrib.seq2seq.BahdanauAttention(\n",
    "                num_units = size_layer, \n",
    "                memory = encoder_outputs,\n",
    "                memory_sequence_length = self.X_seq_len)\n",
    "            att_cell = tf.contrib.seq2seq.AttentionWrapper(\n",
    "                cell = attention_cell,\n",
    "                attention_mechanism = attention_mechanism,\n",
    "                attention_layer_size = None,\n",
    "                alignment_history = True,\n",
    "                output_attention = False)\n",
    "            gcell = GNMTAttentionMultiCell(att_cell, decoder_cells)\n",
    "            \n",
    "            self.initial_state = tuple(\n",
    "                zs.clone(cell_state=es)\n",
    "                if isinstance(zs, tf.contrib.seq2seq.AttentionWrapperState) else es\n",
    "                for zs, es in zip(\n",
    "                    gcell.zero_state(batch_size, dtype=tf.float32), encoder_state))\n",
    "            \n",
    "            training_helper = tf.contrib.seq2seq.TrainingHelper(\n",
    "                decoder_embedded,\n",
    "                self.Y_seq_len,\n",
    "                time_major = False\n",
    "            )\n",
    "            training_decoder = tf.contrib.seq2seq.BasicDecoder(\n",
    "                cell = gcell,\n",
    "                helper = training_helper,\n",
    "                initial_state = self.initial_state,\n",
    "                output_layer = to_dense)\n",
    "            training_decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "                decoder = training_decoder,\n",
    "                impute_finished = True,\n",
    "                maximum_iterations = tf.reduce_max(self.Y_seq_len))\n",
    "            self.training_logits = training_decoder_output.rnn_output\n",
    "            \n",
    "        with tf.variable_scope('decode', reuse=True):\n",
    "            encoder_out_tiled = tf.contrib.seq2seq.tile_batch(encoder_outputs, beam_width)\n",
    "            encoder_state_tiled = tf.contrib.seq2seq.tile_batch(encoder_state, beam_width)\n",
    "            X_seq_len_tiled = tf.contrib.seq2seq.tile_batch(self.X_seq_len, beam_width)\n",
    "            \n",
    "            attention_mechanism = tf.contrib.seq2seq.BahdanauAttention(\n",
    "                num_units = size_layer, \n",
    "                memory = encoder_out_tiled,\n",
    "                memory_sequence_length = X_seq_len_tiled)\n",
    "            att_cell = tf.contrib.seq2seq.AttentionWrapper(\n",
    "                cell = attention_cell,\n",
    "                attention_mechanism = attention_mechanism,\n",
    "                attention_layer_size = None,\n",
    "                alignment_history = False,\n",
    "                output_attention = False)\n",
    "            gcell = GNMTAttentionMultiCell(att_cell, decoder_cells)\n",
    "            \n",
    "            self.initial_state = tuple(\n",
    "                zs.clone(cell_state=es)\n",
    "                if isinstance(zs, tf.contrib.seq2seq.AttentionWrapperState) else es\n",
    "                for zs, es in zip(\n",
    "                    gcell.zero_state(batch_size * beam_width, dtype=tf.float32), encoder_state_tiled))\n",
    "            \n",
    "            predicting_decoder = tf.contrib.seq2seq.BeamSearchDecoder(\n",
    "                cell = gcell,\n",
    "                embedding = embeddings,\n",
    "                start_tokens = tf.tile(tf.constant([GO], dtype=tf.int32), [batch_size]),\n",
    "                end_token = EOS,\n",
    "                initial_state = self.initial_state,\n",
    "                beam_width = beam_width,\n",
    "                output_layer = to_dense,\n",
    "                length_penalty_weight = 0.0)\n",
    "            predicting_decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "                decoder = predicting_decoder,\n",
    "                impute_finished = False,\n",
    "                maximum_iterations = 2 * tf.reduce_max(self.X_seq_len))\n",
    "            self.fast_result = predicting_decoder_output.predicted_ids[:, :, 0]\n",
    "            \n",
    "        masks = tf.sequence_mask(self.Y_seq_len, tf.reduce_max(self.Y_seq_len), dtype=tf.float32)\n",
    "        self.cost = tf.contrib.seq2seq.sequence_loss(logits = self.training_logits,\n",
    "                                                     targets = self.Y,\n",
    "                                                     weights = masks)\n",
    "        self.optimizer = tf.train.AdamOptimizer(learning_rate).minimize(self.cost)\n",
    "        y_t = tf.argmax(self.training_logits,axis=2)\n",
    "        y_t = tf.cast(y_t, tf.int32)\n",
    "        self.prediction = tf.boolean_mask(y_t, masks)\n",
    "        mask_label = tf.boolean_mask(self.Y, masks)\n",
    "        correct_pred = tf.equal(self.prediction, mask_label)\n",
    "        correct_index = tf.cast(correct_pred, tf.float32)\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "size_layer = 512\n",
    "num_layers = 3\n",
    "learning_rate = 1e-3\n",
    "batch_size = 64\n",
    "epoch = 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/client/session.py:1750: UserWarning: An interactive session is already active. This can cause out-of-memory errors in some cases. You must explicitly call `InteractiveSession.close()` to release resources held by the other session(s).\n",
      "  warnings.warn('An interactive session is already active. This can '\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Translator(size_layer, num_layers, size_layer, learning_rate)\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "pad_sequences = tf.keras.preprocessing.sequence.pad_sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([[25165, 25165, 25165, 12719, 12719, 12719, 12719, 11384,  4722,\n",
       "          4722,  4722, 20577, 20577, 20577, 20577, 20577, 31007, 31007,\n",
       "         31007, 31007, 31007, 31007, 31007, 31007, 31007, 31007,    95,\n",
       "            95,    95,    95, 10788, 10788, 26129, 25836, 25836, 25836,\n",
       "         25836, 25836, 25836, 28841, 22661,  6972,  6972,  6972,  6972,\n",
       "          6972,  6972, 18721, 17965, 17965, 17965, 17965, 17965, 20057,\n",
       "         20057,  6549,  6549,  4051,  4051,  4051, 27265, 25803, 25803,\n",
       "         22130,  3280,  3280,  3280, 21261, 21261, 21261, 12184, 12184],\n",
       "        [23674, 22063, 20393, 20393, 25949, 25949, 25949, 25949, 25949,\n",
       "         24464, 24464, 24464, 24464, 24464, 24464,  9336,  9336, 13471,\n",
       "         13471, 13471, 21559, 15741, 30712, 30712, 30712,  3493,  3493,\n",
       "          3493,  3493,  3493,  7100,  7100,  7100,  7100, 24171,  7100,\n",
       "         24171, 24171, 31383, 31383, 31383, 31383, 31383, 31383, 24730,\n",
       "          1435,  1435,  1435,  1435,  1435, 24100, 24100, 17077, 17077,\n",
       "         27546, 27546, 27546, 27546, 27546,  1018, 14427, 14427, 14427,\n",
       "         14427,  1288,  1288,  1288, 12117, 12117, 11043, 12117, 11043],\n",
       "        [ 2799,  2799, 15513, 15513, 17585, 17585, 17585, 17585, 31530,\n",
       "         31530, 31530, 31530, 31530, 31530, 29816, 29816, 29816, 29816,\n",
       "         30605, 30605, 11155, 11155, 25968, 20258, 20258, 20258, 20258,\n",
       "         20258, 24781, 16037, 16037, 16037, 28694, 28694, 28694, 28694,\n",
       "         28694, 13465, 13465, 13465, 13465, 13465,  4367,  4367,  4367,\n",
       "          4367,  4367, 24648, 24648, 24648, 24648, 20209, 20209, 16959,\n",
       "          7939,  7939,  7939,  7939,  7939,  7939,  9431,   884,   884,\n",
       "           884, 23502, 23502, 23502, 16055, 16055, 27778, 27778, 27778],\n",
       "        [23690, 23690, 23466, 23466, 20636, 20636,  6595,  6595,  6595,\n",
       "         11354, 11354, 25826, 25826, 31189, 28645, 28645, 28645, 28645,\n",
       "         28645, 28645, 28645, 16582, 16582, 16582, 16582, 16582, 26439,\n",
       "         26439, 26439, 26439, 26439, 26439,  8884,  7518,  7518,  7518,\n",
       "          7518,  5426,  5426,  5426,  5426,  8135,  8135,    20,    20,\n",
       "            20,    20,    20, 14917, 14917, 14917, 14917, 12198, 12198,\n",
       "         12198, 12198, 12198, 12198, 12198, 10572, 10572, 10572, 19585,\n",
       "         19585, 19585, 19585, 19585, 27849, 19670, 27849, 19670, 19670],\n",
       "        [12009,  1243,  1243,   402,   402,   402,   250,   765, 18652,\n",
       "         20325, 20325, 11693, 11693, 11693, 11693, 10145, 14725, 14725,\n",
       "         14725, 14725, 14725, 17905, 17905, 17905, 17905,  3595,  3595,\n",
       "          3595, 30250, 30250, 30250, 30250, 25109, 25109, 25109, 25109,\n",
       "         25109,  1727,  9160,  9160,  9160,  9160,  9160, 29573, 29573,\n",
       "         29573, 18413, 18413, 18413, 26676, 26676, 29048, 29048,  8339,\n",
       "         10019,  2370,  2370,  2370,  2370, 31397, 31397, 10884, 29898,\n",
       "         29898, 29898, 29898,  6381,  6381,  6381,  6381,  6381, 21203],\n",
       "        [ 6069,  6069, 10613,  2782,  2782,  4022,  5799,  3820,  3820,\n",
       "          3820,   735,   735,   735,   735,  5507,  5507, 22278, 22278,\n",
       "         22278, 22278, 22278, 22278,  3130,  3130,  3130,  3130,  3130,\n",
       "         31193,  4235,  4235,  4235,  4235, 14393,  8583,  8583,  6940,\n",
       "          6940,  6940,  6940,  5542,  5542,  5542, 24201, 24201, 24201,\n",
       "         24201, 11199, 11199,  5072,  5072, 11904, 11904, 11904, 11904,\n",
       "         11904,   381,   381,   381, 26397, 26397, 26397, 30051, 30051,\n",
       "         30051, 20504, 20504, 20504, 31872,  8712, 21364, 21364, 19558],\n",
       "        [24761, 31101, 31101, 31101, 31101, 13779, 13779,  7655,  7655,\n",
       "          7655,  4976,  4976,  4976,  4976,  4976,  4976, 15264,  4881,\n",
       "          4881,  4881,  4881,  4881,  4881,  4881, 11239, 16713, 16713,\n",
       "         16713, 19941, 19941, 19941, 23736, 24570, 24570, 24570, 21996,\n",
       "         15264, 21996, 25539, 21996, 25539, 25539, 10168, 10168, 10168,\n",
       "         21111, 24998, 24998, 27812,  5474,  5474, 29701, 29701, 29701,\n",
       "         26261, 26261, 26261, 14887, 14887, 14887,   504,   504,   504,\n",
       "         21988, 21988, 25783, 28587, 28587, 28587, 25216, 25216, 25216],\n",
       "        [ 9621,  3849,  3849,  3849, 20056, 20056, 22210, 22210, 22210,\n",
       "         22210, 22210,  4889,  4889,  4889,  4889,  4889, 10281, 22904,\n",
       "         22904, 22904, 22904, 22904, 22904, 16934, 10234, 14178, 14178,\n",
       "         14178, 14178,  8964,  9063,  9063,  9063,  9063,  9063,  9063,\n",
       "          9063, 13315, 13315, 13315, 13315, 13315, 13315, 13315, 13315,\n",
       "         13315, 13315, 20625, 20625, 20625, 20625, 11666,  1186,  1186,\n",
       "          1186, 28330,  9362,  9362,  9362, 10730, 10730, 10730, 10730,\n",
       "         10730,  1636,  1636, 10373, 10373, 10373, 10373, 20568, 20935],\n",
       "        [31366, 20724, 20724,  9777, 15993, 15993, 15993, 15993, 31304,\n",
       "         31304, 31304,  3360,  3219,  3219,  3219,  3219,  3219,  3219,\n",
       "          3219, 27794, 27794, 27794, 27794, 27794, 31127, 15901, 15901,\n",
       "         15901, 15901, 15901, 31788, 31788, 31788, 31788, 15953, 15953,\n",
       "         15953,  6993,  6993,   428,   428,   428, 21954, 21954, 21954,\n",
       "         21954, 21954, 21954,  1261,  5045,  5045, 16239, 16376, 16376,\n",
       "         16376, 16376, 16376, 16376, 21742, 23058, 23058, 12370, 12370,\n",
       "         12370, 12370,  8271,  8271,  8271, 11137, 30457, 27482, 27482],\n",
       "        [  647, 11816, 11816,  9976,  9976, 22952, 22952, 22952, 22952,\n",
       "         22952, 16823, 31370, 31370, 31370, 31370, 31370, 14620,  3823,\n",
       "          3823, 28695, 28695, 28695, 28695, 25197,  5007,  5007, 16911,\n",
       "         16911, 16911, 29522, 29522, 29522, 29522, 29522,  1503,  3071,\n",
       "          1503,  3071,  1503, 24004, 24004, 24004,  3800,  3800,  3800,\n",
       "         24549, 21318, 24549, 24549, 24549, 30488, 30488, 30488, 30488,\n",
       "         28521,  1372,  1372,  1372,  1372,   699,   699, 30955, 30955,\n",
       "         23399, 23399, 23399, 13372, 16826, 31950, 31950, 31950,  7102]],\n",
       "       dtype=int32), 10.373663, 0.0]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_x = pad_sequences(train_X[:10], padding='post')\n",
    "batch_y = pad_sequences(train_Y[:10], padding='post')\n",
    "\n",
    "sess.run([model.fast_result, model.cost, model.accuracy], \n",
    "         feed_dict = {model.X: batch_x, model.Y: batch_y})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 3125/3125 [20:10<00:00,  2.58it/s, accuracy=0.256, cost=4.63]\n",
      "minibatch loop: 100%|██████████| 79/79 [00:14<00:00,  5.51it/s, accuracy=0.317, cost=4.11]\n",
      "minibatch loop:   0%|          | 0/3125 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 1, training avg loss 5.345767, training avg acc 0.196012\n",
      "epoch 1, testing avg loss 4.459814, testing avg acc 0.263600\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 3125/3125 [20:29<00:00,  2.54it/s, accuracy=0.313, cost=3.97]\n",
      "minibatch loop: 100%|██████████| 79/79 [00:14<00:00,  5.49it/s, accuracy=0.306, cost=3.66]\n",
      "minibatch loop:   0%|          | 0/3125 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 2, training avg loss 4.084258, training avg acc 0.297619\n",
      "epoch 2, testing avg loss 3.909624, testing avg acc 0.319685\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 3125/3125 [20:36<00:00,  2.53it/s, accuracy=0.353, cost=3.51]\n",
      "minibatch loop: 100%|██████████| 79/79 [00:14<00:00,  5.50it/s, accuracy=0.317, cost=3.48]\n",
      "minibatch loop:   0%|          | 0/3125 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 3, training avg loss 3.528232, training avg acc 0.354670\n",
      "epoch 3, testing avg loss 3.662463, testing avg acc 0.348703\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 3125/3125 [20:36<00:00,  2.53it/s, accuracy=0.389, cost=3.17]\n",
      "minibatch loop: 100%|██████████| 79/79 [00:14<00:00,  5.54it/s, accuracy=0.366, cost=3.39]\n",
      "minibatch loop:   0%|          | 0/3125 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 4, training avg loss 3.134830, training avg acc 0.398924\n",
      "epoch 4, testing avg loss 3.579558, testing avg acc 0.362054\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 3125/3125 [21:37<00:00,  2.41it/s, accuracy=0.417, cost=2.88]\n",
      "minibatch loop: 100%|██████████| 79/79 [00:43<00:00,  1.83it/s, accuracy=0.371, cost=3.32]\n",
      "minibatch loop:   0%|          | 0/3125 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 5, training avg loss 2.828984, training avg acc 0.437010\n",
      "epoch 5, testing avg loss 3.589529, testing avg acc 0.365670\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 3125/3125 [20:52<00:00,  2.49it/s, accuracy=0.459, cost=2.64]\n",
      "minibatch loop: 100%|██████████| 79/79 [00:14<00:00,  5.55it/s, accuracy=0.387, cost=3.33]\n",
      "minibatch loop:   0%|          | 0/3125 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 6, training avg loss 2.574986, training avg acc 0.472001\n",
      "epoch 6, testing avg loss 3.654639, testing avg acc 0.365644\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 3125/3125 [20:03<00:00,  2.60it/s, accuracy=0.496, cost=2.41]\n",
      "minibatch loop: 100%|██████████| 79/79 [00:14<00:00,  5.51it/s, accuracy=0.355, cost=3.42]\n",
      "minibatch loop:   0%|          | 0/3125 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 7, training avg loss 2.359153, training avg acc 0.503841\n",
      "epoch 7, testing avg loss 3.743661, testing avg acc 0.361470\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 3125/3125 [20:15<00:00,  2.57it/s, accuracy=0.523, cost=2.23]\n",
      "minibatch loop: 100%|██████████| 79/79 [00:13<00:00,  5.71it/s, accuracy=0.36, cost=3.46] \n",
      "minibatch loop:   0%|          | 0/3125 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 8, training avg loss 2.172666, training avg acc 0.532985\n",
      "epoch 8, testing avg loss 3.837901, testing avg acc 0.357669\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 3125/3125 [20:24<00:00,  2.55it/s, accuracy=0.554, cost=2.02]\n",
      "minibatch loop: 100%|██████████| 79/79 [00:14<00:00,  5.52it/s, accuracy=0.36, cost=3.5]  \n",
      "minibatch loop:   0%|          | 0/3125 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 9, training avg loss 2.009218, training avg acc 0.559310\n",
      "epoch 9, testing avg loss 3.946232, testing avg acc 0.353705\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 3125/3125 [20:20<00:00,  2.56it/s, accuracy=0.567, cost=1.93]\n",
      "minibatch loop: 100%|██████████| 79/79 [00:14<00:00,  5.54it/s, accuracy=0.349, cost=3.69]\n",
      "minibatch loop:   0%|          | 0/3125 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 10, training avg loss 1.866919, training avg acc 0.582676\n",
      "epoch 10, testing avg loss 4.069591, testing avg acc 0.349418\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 3125/3125 [20:42<00:00,  2.51it/s, accuracy=0.598, cost=1.77]\n",
      "minibatch loop: 100%|██████████| 79/79 [00:13<00:00,  5.65it/s, accuracy=0.339, cost=3.89]\n",
      "minibatch loop:   0%|          | 0/3125 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 11, training avg loss 1.742373, training avg acc 0.603974\n",
      "epoch 11, testing avg loss 4.184957, testing avg acc 0.345629\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 3125/3125 [20:42<00:00,  2.52it/s, accuracy=0.62, cost=1.66] \n",
      "minibatch loop: 100%|██████████| 79/79 [00:14<00:00,  5.50it/s, accuracy=0.355, cost=4.02]\n",
      "minibatch loop:   0%|          | 0/3125 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 12, training avg loss 1.632227, training avg acc 0.622576\n",
      "epoch 12, testing avg loss 4.288281, testing avg acc 0.344445\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 3125/3125 [20:52<00:00,  2.49it/s, accuracy=0.643, cost=1.57]\n",
      "minibatch loop: 100%|██████████| 79/79 [00:14<00:00,  5.56it/s, accuracy=0.376, cost=3.92]\n",
      "minibatch loop:   0%|          | 0/3125 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 13, training avg loss 1.535365, training avg acc 0.639394\n",
      "epoch 13, testing avg loss 4.413867, testing avg acc 0.340973\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 3125/3125 [20:56<00:00,  2.49it/s, accuracy=0.664, cost=1.46]\n",
      "minibatch loop: 100%|██████████| 79/79 [00:14<00:00,  5.55it/s, accuracy=0.349, cost=4.07]\n",
      "minibatch loop:   0%|          | 0/3125 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 14, training avg loss 1.446990, training avg acc 0.655110\n",
      "epoch 14, testing avg loss 4.523855, testing avg acc 0.340079\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 3125/3125 [20:59<00:00,  2.48it/s, accuracy=0.67, cost=1.4]   \n",
      "minibatch loop: 100%|██████████| 79/79 [00:14<00:00,  5.49it/s, accuracy=0.333, cost=4.01]\n",
      "minibatch loop:   0%|          | 0/3125 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 15, training avg loss 1.367731, training avg acc 0.669252\n",
      "epoch 15, testing avg loss 4.623041, testing avg acc 0.337174\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 3125/3125 [21:03<00:00,  2.47it/s, accuracy=0.682, cost=1.3]  \n",
      "minibatch loop: 100%|██████████| 79/79 [00:14<00:00,  5.47it/s, accuracy=0.36, cost=4.2]  \n",
      "minibatch loop:   0%|          | 0/3125 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 16, training avg loss 1.298758, training avg acc 0.681641\n",
      "epoch 16, testing avg loss 4.742639, testing avg acc 0.336234\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 3125/3125 [21:00<00:00,  2.48it/s, accuracy=0.678, cost=1.29] \n",
      "minibatch loop: 100%|██████████| 79/79 [00:14<00:00,  5.63it/s, accuracy=0.382, cost=4.15]\n",
      "minibatch loop:   0%|          | 0/3125 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 17, training avg loss 1.237175, training avg acc 0.692671\n",
      "epoch 17, testing avg loss 4.843971, testing avg acc 0.334623\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 3125/3125 [20:48<00:00,  2.50it/s, accuracy=0.699, cost=1.22] \n",
      "minibatch loop: 100%|██████████| 79/79 [00:14<00:00,  5.60it/s, accuracy=0.339, cost=4.31]\n",
      "minibatch loop:   0%|          | 0/3125 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 18, training avg loss 1.182478, training avg acc 0.702417\n",
      "epoch 18, testing avg loss 4.944494, testing avg acc 0.330336\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 3125/3125 [20:49<00:00,  2.50it/s, accuracy=0.72, cost=1.11]  \n",
      "minibatch loop: 100%|██████████| 79/79 [00:14<00:00,  5.60it/s, accuracy=0.382, cost=4.28]\n",
      "minibatch loop:   0%|          | 0/3125 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 19, training avg loss 1.132978, training avg acc 0.711609\n",
      "epoch 19, testing avg loss 5.042641, testing avg acc 0.330751\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 3125/3125 [20:49<00:00,  2.50it/s, accuracy=0.727, cost=1.09] \n",
      "minibatch loop: 100%|██████████| 79/79 [00:14<00:00,  5.59it/s, accuracy=0.382, cost=4.28]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 20, training avg loss 1.088114, training avg acc 0.719840\n",
      "epoch 20, testing avg loss 5.144392, testing avg acc 0.328790\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "import tqdm\n",
    "\n",
    "for e in range(epoch):\n",
    "    pbar = tqdm.tqdm(\n",
    "        range(0, len(train_X), batch_size), desc = 'minibatch loop')\n",
    "    train_loss, train_acc, test_loss, test_acc = [], [], [], []\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(train_X))\n",
    "        batch_x = pad_sequences(train_X[i : index], padding='post')\n",
    "        batch_y = pad_sequences(train_Y[i : index], padding='post')\n",
    "        feed = {model.X: batch_x,\n",
    "                model.Y: batch_y}\n",
    "        accuracy, loss, _ = sess.run([model.accuracy,model.cost,model.optimizer],\n",
    "                                    feed_dict = feed)\n",
    "        train_loss.append(loss)\n",
    "        train_acc.append(accuracy)\n",
    "        pbar.set_postfix(cost = loss, accuracy = accuracy)\n",
    "    \n",
    "    \n",
    "    pbar = tqdm.tqdm(\n",
    "        range(0, len(test_X), batch_size), desc = 'minibatch loop')\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(test_X))\n",
    "        batch_x = pad_sequences(test_X[i : index], padding='post')\n",
    "        batch_y = pad_sequences(test_Y[i : index], padding='post')\n",
    "        feed = {model.X: batch_x,\n",
    "                model.Y: batch_y,}\n",
    "        accuracy, loss = sess.run([model.accuracy,model.cost],\n",
    "                                    feed_dict = feed)\n",
    "\n",
    "        test_loss.append(loss)\n",
    "        test_acc.append(accuracy)\n",
    "        pbar.set_postfix(cost = loss, accuracy = accuracy)\n",
    "    \n",
    "    print('epoch %d, training avg loss %f, training avg acc %f'%(e+1,\n",
    "                                                                 np.mean(train_loss),np.mean(train_acc)))\n",
    "    print('epoch %d, testing avg loss %f, testing avg acc %f'%(e+1,\n",
    "                                                              np.mean(test_loss),np.mean(test_acc)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensor2tensor.utils import bleu_hook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 79/79 [00:34<00:00,  2.30it/s]\n"
     ]
    }
   ],
   "source": [
    "results = []\n",
    "for i in tqdm.tqdm(range(0, len(test_X), batch_size)):\n",
    "    index = min(i + batch_size, len(test_X))\n",
    "    batch_x = pad_sequences(test_X[i : index], padding='post')\n",
    "    feed = {model.X: batch_x}\n",
    "    p = sess.run(model.fast_result,feed_dict = feed)\n",
    "    result = []\n",
    "    for row in p:\n",
    "        result.append([i for i in row if i > 3])\n",
    "    results.extend(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "rights = []\n",
    "for r in test_Y:\n",
    "    rights.append([i for i in r if i > 3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.055380445"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bleu_hook.compute_bleu(reference_corpus = rights,\n",
    "                       translation_corpus = results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}