mt.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Seq2Seq NMT\n",
    "The following code will translate English-Vietnamese using NMT. \n",
    "I have used code provided at https://github.com/tensorflow/nmt, proceed to the link for details on Seq2Seq architecture. \n",
    "In this code, i am using Luong-scaled attention mechanism [Luong et al., 2015](https://arxiv.org/abs/1508.04025)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-27T18:31:13.051081Z",
     "start_time": "2017-07-27T18:31:11.455535Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading Hyper-parameters \n",
    "The HParams class will hold all Hyper-parameters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-27T18:31:20.684330Z",
     "start_time": "2017-07-27T18:31:13.223245Z"
    }
   },
   "outputs": [],
   "source": [
    "hparams = tf.contrib.training.HParams()\n",
    "data=''\n",
    "with open (\"params.json\", \"r\") as jfile:\n",
    "    data=eval(jfile.read())\n",
    "for param,value in data.items():\n",
    "    hparams.add_hparam(param,value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-27T18:31:20.870301Z",
     "start_time": "2017-07-27T18:31:20.865556Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "hparams.add_hparam('num_residual_layers',0) # Not using it!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-27T18:31:21.073857Z",
     "start_time": "2017-07-27T18:31:21.063823Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#from https://stackoverflow.com/a/38580201/6077501\n",
    "from tensorflow.python.client import device_lib\n",
    "\n",
    "def get_available_gpus():\n",
    "    local_device_protos = device_lib.list_local_devices()\n",
    "    return len([x.name for x in local_device_protos if x.device_type == 'GPU'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-27T18:31:21.674678Z",
     "start_time": "2017-07-27T18:31:21.263939Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "hparams.add_hparam('num_gpus',get_available_gpus())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pre-processing "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-27T18:31:21.861793Z",
     "start_time": "2017-07-27T18:31:21.856527Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import vocab_utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-27T18:31:22.231293Z",
     "start_time": "2017-07-27T18:31:22.069433Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "b'# Vocab file /data/manni/mt_data/vocab.en exists'\n",
      "b'# Vocab file /data/manni/mt_data/vocab.vi exists'\n"
     ]
    }
   ],
   "source": [
    "src_vocab_size, _ = vocab_utils.check_vocab(hparams.src_vocab_file,hparams.out_dir)\n",
    "tgt_vocab_size, _ = vocab_utils.check_vocab(hparams.tgt_vocab_file,hparams.out_dir)\n",
    "hparams.add_hparam('src_vocab_size',src_vocab_size)\n",
    "hparams.add_hparam('tgt_vocab_size',tgt_vocab_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-27T18:31:22.450398Z",
     "start_time": "2017-07-27T18:31:22.444903Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "out_dir=hparams.out_dir\n",
    "if not tf.gfile.Exists(out_dir): tf.gfile.MakeDirs(out_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Creator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-27T18:31:22.674252Z",
     "start_time": "2017-07-27T18:31:22.660603Z"
    }
   },
   "outputs": [],
   "source": [
    "import attention_model\n",
    "import model\n",
    "import model_helper\n",
    "import iterator_utils\n",
    "import collections\n",
    "import misc_utils as utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-27T18:31:22.889377Z",
     "start_time": "2017-07-27T18:31:22.885229Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model_type=attention_model.AttentionModel"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Train Graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-27T18:31:23.336285Z",
     "start_time": "2017-07-27T18:31:23.108240Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "class TrainGraph(collections.namedtuple(\"TrainGraph\", (\"graph\", \"model\", \"iterator\",\"skip_count_placeholder\"))):\n",
    "    pass\n",
    "\n",
    "\n",
    "def create_train_graph(scope=None):\n",
    "    graph = tf.Graph()\n",
    "    with graph.as_default():\n",
    "        src_vocab_table, tgt_vocab_table = vocab_utils.create_vocab_tables(hparams.src_vocab_file, \n",
    "                                                                           hparams.tgt_vocab_file)\n",
    "        \n",
    "        src_dataset = tf.contrib.data.TextLineDataset(hparams.src_file)\n",
    "        tgt_dataset = tf.contrib.data.TextLineDataset(hparams.tgt_file)\n",
    "        skip_count_placeholder = tf.placeholder(shape=(), dtype=tf.int64)  # Needed?\n",
    "        \n",
    "        iterator = iterator_utils.get_iterator(src_dataset,tgt_dataset,src_vocab_table,tgt_vocab_table,\n",
    "            batch_size=hparams.batch_size,\n",
    "            sos=hparams.sos,\n",
    "            eos=hparams.eos,\n",
    "            source_reverse=hparams.source_reverse,\n",
    "            random_seed=None,\n",
    "            num_buckets=hparams.num_buckets,\n",
    "            src_max_len=hparams.src_max_len,\n",
    "            tgt_max_len=hparams.tgt_max_len,\n",
    "            skip_count=skip_count_placeholder)\n",
    "        \n",
    "        model = model_type(hparams,\n",
    "                iterator=iterator,\n",
    "                mode=tf.contrib.learn.ModeKeys.TRAIN,\n",
    "                source_vocab_table=src_vocab_table,\n",
    "                target_vocab_table=tgt_vocab_table,\n",
    "                scope=scope)\n",
    "        \n",
    "    return TrainGraph(graph=graph,model=model,iterator=iterator,skip_count_placeholder=skip_count_placeholder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-27T18:31:27.943743Z",
     "start_time": "2017-07-27T18:31:23.554707Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "b'# creating train graph ...'\n",
      "b'  num_bi_layers = 1, num_bi_residual_layers=0'\n",
      "b'  cell 0'b'  LSTM, forget_bias=1'b'  DropoutWrapper, dropout=0.2 'b'  DeviceWrapper, device=/gpu:0'b''\n",
      "b'  cell 0'b'  LSTM, forget_bias=1'b'  DropoutWrapper, dropout=0.2 'b'  DeviceWrapper, device=/gpu:0'b''\n",
      "b'  cell 0'b'  LSTM, forget_bias=1'b'  DropoutWrapper, dropout=0.2 'b'  DeviceWrapper, device=/gpu:0'b''\n",
      "b'  cell 1'b'  LSTM, forget_bias=1'b'  DropoutWrapper, dropout=0.2 'b'  DeviceWrapper, device=/gpu:0'b''\n",
      "  start_decay_step=8000, learning_rate=1, decay_steps 1000,decay_factor 0.5\n",
      "b'# Trainable variables'\n",
      "b'  embeddings/encoder/embedding_encoder:0, (17191, 512), '\n",
      "b'  embeddings/decoder/embedding_decoder:0, (7709, 512), '\n",
      "b'  dynamic_seq2seq/encoder/bidirectional_rnn/fw/basic_lstm_cell/kernel:0, (1024, 2048), /device:GPU:0'\n",
      "b'  dynamic_seq2seq/encoder/bidirectional_rnn/fw/basic_lstm_cell/bias:0, (2048,), /device:GPU:0'\n",
      "b'  dynamic_seq2seq/encoder/bidirectional_rnn/bw/basic_lstm_cell/kernel:0, (1024, 2048), /device:GPU:0'\n",
      "b'  dynamic_seq2seq/encoder/bidirectional_rnn/bw/basic_lstm_cell/bias:0, (2048,), /device:GPU:0'\n",
      "b'  dynamic_seq2seq/decoder/memory_layer/kernel:0, (1024, 512), '\n",
      "b'  dynamic_seq2seq/decoder/attention/multi_rnn_cell/cell_0/basic_lstm_cell/kernel:0, (1536, 2048), /device:GPU:0'\n",
      "b'  dynamic_seq2seq/decoder/attention/multi_rnn_cell/cell_0/basic_lstm_cell/bias:0, (2048,), /device:GPU:0'\n",
      "b'  dynamic_seq2seq/decoder/attention/multi_rnn_cell/cell_1/basic_lstm_cell/kernel:0, (1024, 2048), /device:GPU:0'\n",
      "b'  dynamic_seq2seq/decoder/attention/multi_rnn_cell/cell_1/basic_lstm_cell/bias:0, (2048,), /device:GPU:0'\n",
      "b'  dynamic_seq2seq/decoder/attention/luong_attention/attention_g:0, (), /device:GPU:0'\n",
      "b'  dynamic_seq2seq/decoder/attention/attention_layer/kernel:0, (1536, 512), /device:GPU:0'\n",
      "b'  dynamic_seq2seq/decoder/output_projection/kernel:0, (512, 7709), /device:GPU:0'\n"
     ]
    }
   ],
   "source": [
    "train_graph=create_train_graph()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Eval Graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-27T18:31:28.051847Z",
     "start_time": "2017-07-27T18:31:28.033551Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "class EvalGraph(collections.namedtuple(\"EvalGraph\",(\"graph\", \"model\", \"src_file_placeholder\",\n",
    "                                                    \"tgt_file_placeholder\", \n",
    "                                                    \"iterator\"))):\n",
    "    pass\n",
    "\n",
    "def create_eval_graph(scope=None):\n",
    "    graph = tf.Graph()\n",
    "\n",
    "    with graph.as_default():\n",
    "        src_vocab_table, tgt_vocab_table = vocab_utils.create_vocab_tables(\n",
    "            hparams.src_vocab_file, hparams.tgt_vocab_file, hparams.share_vocab)\n",
    "        src_file_placeholder = tf.placeholder(shape=(), dtype=tf.string)\n",
    "        tgt_file_placeholder = tf.placeholder(shape=(), dtype=tf.string)\n",
    "        src_dataset = tf.contrib.data.TextLineDataset(src_file_placeholder)\n",
    "        tgt_dataset = tf.contrib.data.TextLineDataset(tgt_file_placeholder)\n",
    "        iterator = iterator_utils.get_iterator(\n",
    "            src_dataset,\n",
    "            tgt_dataset,\n",
    "            src_vocab_table,\n",
    "            tgt_vocab_table,\n",
    "            hparams.batch_size,\n",
    "            sos=hparams.sos,\n",
    "            eos=hparams.eos,\n",
    "            source_reverse=hparams.source_reverse,\n",
    "            random_seed=hparams.random_seed,\n",
    "            num_buckets=hparams.num_buckets,\n",
    "            src_max_len=hparams.src_max_len_infer,\n",
    "            tgt_max_len=hparams.tgt_max_len_infer)\n",
    "        model = model_type(\n",
    "            hparams,\n",
    "            iterator=iterator,\n",
    "            mode=tf.contrib.learn.ModeKeys.EVAL,\n",
    "            source_vocab_table=src_vocab_table,\n",
    "            target_vocab_table=tgt_vocab_table,\n",
    "            scope=scope)\n",
    "    return EvalGraph(graph=graph,model=model,src_file_placeholder=src_file_placeholder,\n",
    "                tgt_file_placeholder=tgt_file_placeholder,iterator=iterator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-27T18:31:30.438128Z",
     "start_time": "2017-07-27T18:31:28.280896Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "b'# creating eval graph ...'\n",
      "b'  num_bi_layers = 1, num_bi_residual_layers=0'\n",
      "b'  cell 0'b'  LSTM, forget_bias=1'b'  DeviceWrapper, device=/gpu:0'b''\n",
      "b'  cell 0'b'  LSTM, forget_bias=1'b'  DeviceWrapper, device=/gpu:0'b''\n",
      "b'  cell 0'b'  LSTM, forget_bias=1'b'  DeviceWrapper, device=/gpu:0'b''\n",
      "b'  cell 1'b'  LSTM, forget_bias=1'b'  DeviceWrapper, device=/gpu:0'b''\n",
      "  start_decay_step=8000, learning_rate=1, decay_steps 1000,decay_factor 0.5\n",
      "b'# Trainable variables'\n",
      "b'  embeddings/encoder/embedding_encoder:0, (17191, 512), '\n",
      "b'  embeddings/decoder/embedding_decoder:0, (7709, 512), '\n",
      "b'  dynamic_seq2seq/encoder/bidirectional_rnn/fw/basic_lstm_cell/kernel:0, (1024, 2048), /device:GPU:0'\n",
      "b'  dynamic_seq2seq/encoder/bidirectional_rnn/fw/basic_lstm_cell/bias:0, (2048,), /device:GPU:0'\n",
      "b'  dynamic_seq2seq/encoder/bidirectional_rnn/bw/basic_lstm_cell/kernel:0, (1024, 2048), /device:GPU:0'\n",
      "b'  dynamic_seq2seq/encoder/bidirectional_rnn/bw/basic_lstm_cell/bias:0, (2048,), /device:GPU:0'\n",
      "b'  dynamic_seq2seq/decoder/memory_layer/kernel:0, (1024, 512), '\n",
      "b'  dynamic_seq2seq/decoder/attention/multi_rnn_cell/cell_0/basic_lstm_cell/kernel:0, (1536, 2048), /device:GPU:0'\n",
      "b'  dynamic_seq2seq/decoder/attention/multi_rnn_cell/cell_0/basic_lstm_cell/bias:0, (2048,), /device:GPU:0'\n",
      "b'  dynamic_seq2seq/decoder/attention/multi_rnn_cell/cell_1/basic_lstm_cell/kernel:0, (1024, 2048), /device:GPU:0'\n",
      "b'  dynamic_seq2seq/decoder/attention/multi_rnn_cell/cell_1/basic_lstm_cell/bias:0, (2048,), /device:GPU:0'\n",
      "b'  dynamic_seq2seq/decoder/attention/luong_attention/attention_g:0, (), /device:GPU:0'\n",
      "b'  dynamic_seq2seq/decoder/attention/attention_layer/kernel:0, (1536, 512), /device:GPU:0'\n",
      "b'  dynamic_seq2seq/decoder/output_projection/kernel:0, (512, 7709), /device:GPU:0'\n"
     ]
    }
   ],
   "source": [
    "# see the graph\n",
    "eval_graph=create_eval_graph()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Inference Graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-27T18:31:30.614808Z",
     "start_time": "2017-07-27T18:31:30.580168Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "from tensorflow.python.ops import lookup_ops\n",
    "\n",
    "class InferGraph(\n",
    "    collections.namedtuple(\"InferGraph\",(\"graph\", \"model\", \"src_placeholder\",\"batch_size_placeholder\", \n",
    "                                         \"iterator\"))):\n",
    "    pass\n",
    "\n",
    "def create_infer_graph(scope=None):\n",
    "    graph = tf.Graph()\n",
    "    with graph.as_default():\n",
    "        src_vocab_table, tgt_vocab_table = vocab_utils.create_vocab_tables(hparams.src_vocab_file, \n",
    "                                                                           hparams.tgt_vocab_file, \n",
    "                                                                           hparams.share_vocab)\n",
    "        reverse_tgt_vocab_table = lookup_ops.index_to_string_table_from_file(hparams.tgt_vocab_file, \n",
    "                                                                             default_value=vocab_utils.UNK)\n",
    "\n",
    "        src_placeholder = tf.placeholder(shape=[None], dtype=tf.string)\n",
    "        batch_size_placeholder = tf.placeholder(shape=[], dtype=tf.int64)\n",
    "        src_dataset = tf.contrib.data.Dataset.from_tensor_slices(src_placeholder)\n",
    "        iterator = iterator_utils.get_infer_iterator(src_dataset,\n",
    "                                                     src_vocab_table,\n",
    "                                                     batch_size=batch_size_placeholder,\n",
    "                                                     eos=hparams.eos,\n",
    "                                                     source_reverse=hparams.source_reverse,\n",
    "                                                     src_max_len=hparams.src_max_len_infer)\n",
    "        model = model_type(hparams,\n",
    "                           iterator=iterator,\n",
    "                           mode=tf.contrib.learn.ModeKeys.INFER,\n",
    "                           source_vocab_table=src_vocab_table,\n",
    "                           target_vocab_table=tgt_vocab_table,\n",
    "                           reverse_target_vocab_table=reverse_tgt_vocab_table,\n",
    "                           scope=scope)\n",
    "    return InferGraph(graph=graph,model=model,src_placeholder=src_placeholder,\n",
    "                      batch_size_placeholder=batch_size_placeholder,iterator=iterator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-27T18:31:33.384291Z",
     "start_time": "2017-07-27T18:31:30.805276Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "b'# creating infer graph ...'\n",
      "b'  num_bi_layers = 1, num_bi_residual_layers=0'\n",
      "b'  cell 0'b'  LSTM, forget_bias=1'b'  DeviceWrapper, device=/gpu:0'b''\n",
      "b'  cell 0'b'  LSTM, forget_bias=1'b'  DeviceWrapper, device=/gpu:0'b''\n",
      "b'  cell 0'b'  LSTM, forget_bias=1'b'  DeviceWrapper, device=/gpu:0'b''\n",
      "b'  cell 1'b'  LSTM, forget_bias=1'b'  DeviceWrapper, device=/gpu:0'b''\n",
      "  start_decay_step=8000, learning_rate=1, decay_steps 1000,decay_factor 0.5\n",
      "b'# Trainable variables'\n",
      "b'  embeddings/encoder/embedding_encoder:0, (17191, 512), '\n",
      "b'  embeddings/decoder/embedding_decoder:0, (7709, 512), '\n",
      "b'  dynamic_seq2seq/encoder/bidirectional_rnn/fw/basic_lstm_cell/kernel:0, (1024, 2048), /device:GPU:0'\n",
      "b'  dynamic_seq2seq/encoder/bidirectional_rnn/fw/basic_lstm_cell/bias:0, (2048,), /device:GPU:0'\n",
      "b'  dynamic_seq2seq/encoder/bidirectional_rnn/bw/basic_lstm_cell/kernel:0, (1024, 2048), /device:GPU:0'\n",
      "b'  dynamic_seq2seq/encoder/bidirectional_rnn/bw/basic_lstm_cell/bias:0, (2048,), /device:GPU:0'\n",
      "b'  dynamic_seq2seq/decoder/memory_layer/kernel:0, (1024, 512), '\n",
      "b'  dynamic_seq2seq/decoder/attention/multi_rnn_cell/cell_0/basic_lstm_cell/kernel:0, (1536, 2048), /device:GPU:0'\n",
      "b'  dynamic_seq2seq/decoder/attention/multi_rnn_cell/cell_0/basic_lstm_cell/bias:0, (2048,), /device:GPU:0'\n",
      "b'  dynamic_seq2seq/decoder/attention/multi_rnn_cell/cell_1/basic_lstm_cell/kernel:0, (1024, 2048), /device:GPU:0'\n",
      "b'  dynamic_seq2seq/decoder/attention/multi_rnn_cell/cell_1/basic_lstm_cell/bias:0, (2048,), /device:GPU:0'\n",
      "b'  dynamic_seq2seq/decoder/attention/luong_attention/attention_g:0, (), /device:GPU:0'\n",
      "b'  dynamic_seq2seq/decoder/attention/attention_layer/kernel:0, (1536, 512), /device:GPU:0'\n",
      "b'  dynamic_seq2seq/decoder/output_projection/kernel:0, (512, 7709), '\n"
     ]
    }
   ],
   "source": [
    "# see the graph\n",
    "infer_graph=create_infer_graph()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Data loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-27T18:31:33.544664Z",
     "start_time": "2017-07-27T18:31:33.535914Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "import codecs\n",
    "\n",
    "def load_data(inference_input_file):\n",
    "    with codecs.getreader(\"utf-8\")(tf.gfile.GFile(inference_input_file, mode=\"rb\")) as f:\n",
    "        inference_data = f.read().splitlines()\n",
    "    if hparams and hparams.inference_indices:\n",
    "        inference_data = [inference_data[i] for i in hparams.inference_indices]\n",
    "    return inference_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### Loading a sample from Eval set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-27T18:31:33.774784Z",
     "start_time": "2017-07-27T18:31:33.762290Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "sample_src_data = load_data(hparams.dev_src_file)\n",
    "sample_tgt_data = load_data(hparams.dev_tgt_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### Assign sessions to graphs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-27T18:31:34.111730Z",
     "start_time": "2017-07-27T18:31:34.097263Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "config_proto = utils.get_config_proto(log_device_placement=True)\n",
    "\n",
    "train_sess = tf.Session(config=config_proto, graph=train_graph.graph)\n",
    "eval_sess = tf.Session(config=config_proto, graph=eval_graph.graph)\n",
    "infer_sess = tf.Session(config=config_proto, graph=infer_graph.graph)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### Load Train Graph to Process"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-27T18:31:36.527646Z",
     "start_time": "2017-07-27T18:31:34.448639Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "b'  created train model with fresh parameters, time 2.06s'\n"
     ]
    }
   ],
   "source": [
    "with train_graph.graph.as_default():\n",
    "    loaded_train_model, global_step = model_helper.create_or_load_model(train_graph.model, \n",
    "                                                                        hparams.out_dir, \n",
    "                                                                        train_sess, \n",
    "                                                                        \"train\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### Write Graph to TensorBoard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-27T18:31:40.786786Z",
     "start_time": "2017-07-27T18:31:36.762236Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "summary_writer = tf.summary.FileWriter(os.path.join(hparams.out_dir, 'Training'), train_graph.graph)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Test on a random sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-27T18:31:40.931076Z",
     "start_time": "2017-07-27T18:31:40.926928Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "for metric in hparams.metrics:\n",
    "    hparams.add_hparam(\"best_\" + metric, 0)  \n",
    "    best_metric_dir = os.path.join(hparams.out_dir, \"best_\" + metric)\n",
    "    hparams.add_hparam(\"best_\" + metric + \"_dir\", best_metric_dir)\n",
    "    tf.gfile.MakeDirs(best_metric_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-27T18:31:46.899567Z",
     "start_time": "2017-07-27T18:31:41.082433Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "b'  created infer model with fresh parameters, time 1.20s'\n",
      "b'  # 1501'\n",
      "    src: This is Bjorn Sundin .\n",
      "    ref: Bjorn Sundin .\n",
      "    nmt: 100 đúc đúc đúc đúc Hirshhorn Hirshhorn Lai Lai Lai\n",
      "b'  created eval model with fresh parameters, time 0.68s'\n",
      "  eval dev: perplexity 8363.03, time 3s, Thu Jul 27 19:31:46 2017.\n",
      "b'  created infer model with fresh parameters, time 0.30s'\n"
     ]
    }
   ],
   "source": [
    "import Train\n",
    "\n",
    "#Initial evaluation\n",
    "eval_results, _ = Train.run_full_eval(hparams.out_dir, infer_graph, infer_sess, eval_graph, eval_sess, \n",
    "                                      hparams, summary_writer, sample_src_data, sample_tgt_data)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Intialization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-27T18:31:55.881581Z",
     "start_time": "2017-07-27T18:31:55.876639Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-27T18:31:56.357915Z",
     "start_time": "2017-07-27T18:31:56.285895Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "b'# Start step 0, lr 1, Thu Jul 27 19:31:56 2017'\n",
      "b'# Init train iterator, skipping 0 elements'\n"
     ]
    }
   ],
   "source": [
    "last_stats_step = global_step\n",
    "last_eval_step = global_step\n",
    "last_external_eval_step = global_step\n",
    "\n",
    "steps_per_eval = 10 * hparams.steps_per_stats\n",
    "steps_per_external_eval = 5 * steps_per_eval\n",
    "\n",
    "avg_step_time = 0.0\n",
    "step_time, checkpoint_loss, checkpoint_predict_count = 0.0, 0.0, 0.0\n",
    "checkpoint_total_count = 0.0\n",
    "speed, train_ppl = 0.0, 0.0\n",
    "start_train_time = time.time()\n",
    "\n",
    "utils.print_out(\"# Start step %d, lr %g, %s\" %\n",
    "                (global_step, loaded_train_model.learning_rate.eval(session=train_sess),\n",
    "                 time.ctime()))\n",
    "skip_count = hparams.batch_size * hparams.epoch_step\n",
    "utils.print_out(\"# Init train iterator, skipping %d elements\" % skip_count)\n",
    "\n",
    "train_sess.run(train_graph.iterator.initializer,\n",
    "              feed_dict={train_graph.skip_count_placeholder: skip_count})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-27T20:35:01.603124Z",
     "start_time": "2017-07-27T18:31:58.037541Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "b'  global step 100 lr 1 step-time 0.56s wps 9.97K ppl 598274.04 bleu 0.00'\n",
      "b'  global step 200 lr 1 step-time 0.51s wps 10.93K ppl 63224.74 bleu 0.00'\n",
      "b'  global step 300 lr 1 step-time 0.52s wps 10.88K ppl 5383.52 bleu 0.00'\n",
      "b'  global step 400 lr 1 step-time 0.51s wps 10.95K ppl 976.09 bleu 0.00'\n",
      "b'  global step 500 lr 1 step-time 0.52s wps 10.92K ppl 569.79 bleu 0.00'\n",
      "b'  global step 600 lr 1 step-time 0.52s wps 10.96K ppl 433.69 bleu 0.00'\n",
      "b'  global step 700 lr 1 step-time 0.51s wps 11.00K ppl 335.05 bleu 0.00'\n",
      "b'  global step 800 lr 1 step-time 0.51s wps 11.04K ppl 252.46 bleu 0.00'\n",
      "b'  global step 900 lr 1 step-time 0.51s wps 11.04K ppl 201.56 bleu 0.00'\n",
      "b'  global step 1000 lr 1 step-time 0.51s wps 11.00K ppl 156.57 bleu 0.00'\n",
      "b'# Save eval, global step 1000'\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-1000\n",
      "b'  loaded infer model parameters from /data/manni/mt_model/translate.ckpt-1000, time 0.12s'\n",
      "b'  # 95'\n",
      "    src: &quot; It can &apos;t be done &quot; was shown to be wrong .\n",
      "    ref: &quot; Không làm được đâu &quot; bị chứng minh là sai .\n",
      "    nmt: &quot; . </s> <unk> <unk> <unk> <unk> <unk>\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-1000\n",
      "b'  loaded eval model parameters from /data/manni/mt_model/translate.ckpt-1000, time 0.18s'\n",
      "  eval dev: perplexity 131.58, time 3s, Thu Jul 27 19:40:42 2017.\n",
      "b'# Finished an epoch, step 1043. Perform external evaluation'\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-1000\n",
      "b'  loaded infer model parameters from /data/manni/mt_model/translate.ckpt-1000, time 0.23s'\n",
      "b'  # 387'\n",
      "    src: And more than anything , for those of you who share that , I &apos;ve simply come to tell you to keep your eyes on the prize , hold on .\n",
      "    ref: Hơn hết , với những điều này tôi chỉ đơn giản nói với các bạn là hãy theo đuổi mục tiêu của mình đến cùng , hãy tiếp tục phát huy .\n",
      "    nmt: Và nhiều hơn . </s> <unk> <unk>\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-1000\n",
      "b'  loaded infer model parameters from /data/manni/mt_model/translate.ckpt-1000, time 0.22s'\n",
      "b'# External evaluation, global step 1000'\n",
      "b'  decoding to output /data/manni/mt_model/output_dev.'\n",
      "  done, num sentences 1553, time 57s, Thu Jul 27 19:42:02 2017.\n",
      "b'  bleu dev: 0.0'\n",
      "b'  saving hparams to /data/manni/mt_model/hparams'\n",
      "b'  global step 1100 lr 1 step-time 0.56s wps 9.74K ppl 122.87 bleu 0.00'\n",
      "b'  global step 1200 lr 1 step-time 0.53s wps 10.57K ppl 92.70 bleu 0.00'\n",
      "b'  global step 1300 lr 1 step-time 0.52s wps 10.99K ppl 73.80 bleu 0.00'\n",
      "b'  global step 1400 lr 1 step-time 0.51s wps 10.98K ppl 57.53 bleu 0.00'\n",
      "b'  global step 1500 lr 1 step-time 0.51s wps 11.10K ppl 48.50 bleu 0.00'\n",
      "b'  global step 1600 lr 1 step-time 0.52s wps 11.08K ppl 40.97 bleu 0.00'\n",
      "b'  global step 1700 lr 1 step-time 0.51s wps 11.03K ppl 35.69 bleu 0.00'\n",
      "b'  global step 1800 lr 1 step-time 0.51s wps 11.02K ppl 31.14 bleu 0.00'\n",
      "b'  global step 1900 lr 1 step-time 0.51s wps 11.01K ppl 28.44 bleu 0.00'\n",
      "b'  global step 2000 lr 1 step-time 0.50s wps 11.17K ppl 26.08 bleu 0.00'\n",
      "b'# Save eval, global step 2000'\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-2000\n",
      "b'  loaded infer model parameters from /data/manni/mt_model/translate.ckpt-2000, time 0.33s'\n",
      "b'  # 637'\n",
      "    src: How engaged you are .\n",
      "    ref: Bạn thật bận rộn đó .\n",
      "    nmt: Làm như thế nào . </s> </s> <unk> <unk> <unk>\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-2000\n",
      "b'  loaded eval model parameters from /data/manni/mt_model/translate.ckpt-2000, time 0.26s'\n",
      "  eval dev: perplexity 26.39, time 3s, Thu Jul 27 19:50:24 2017.\n",
      "b'# Finished an epoch, step 2086. Perform external evaluation'\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-2000\n",
      "b'  loaded infer model parameters from /data/manni/mt_model/translate.ckpt-2000, time 0.20s'\n",
      "b'  # 1456'\n",
      "    src: Instead of having to program , to wire , to solder , littleBits allow you to program using very simple intuitive gestures .\n",
      "    ref: Thay vì phải lập trình , nối dây hay hàn mạch , littleBits cho phép bạn lập trình bằng những thao tác trực quan đơn giản .\n",
      "    nmt: Thay vì để chương trình cho chương trình , đối với <unk> , <unk> <unk> cho phép các bạn để chương trình sử dụng đơn giản đơn giản </s> </s> </s> <unk> <unk>\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-2000\n",
      "b'  loaded infer model parameters from /data/manni/mt_model/translate.ckpt-2000, time 0.20s'\n",
      "b'# External evaluation, global step 2000'\n",
      "b'  decoding to output /data/manni/mt_model/output_dev.'\n",
      "  done, num sentences 1553, time 116s, Thu Jul 27 19:53:05 2017.\n",
      "b'  bleu dev: 0.0'\n",
      "b'  saving hparams to /data/manni/mt_model/hparams'\n",
      "b'  global step 2100 lr 1 step-time 0.52s wps 10.52K ppl 24.00 bleu 0.00'\n",
      "b'  global step 2200 lr 1 step-time 0.55s wps 9.96K ppl 20.61 bleu 0.00'\n",
      "b'  global step 2300 lr 1 step-time 0.51s wps 11.27K ppl 20.12 bleu 0.00'\n",
      "b'  global step 2400 lr 1 step-time 0.51s wps 11.05K ppl 19.00 bleu 0.00'\n",
      "b'  global step 2500 lr 1 step-time 0.50s wps 11.17K ppl 18.15 bleu 0.00'\n",
      "b'  global step 2600 lr 1 step-time 0.51s wps 11.21K ppl 17.71 bleu 0.00'\n",
      "b'  global step 2700 lr 1 step-time 0.51s wps 11.09K ppl 17.09 bleu 0.00'\n",
      "b'  global step 2800 lr 1 step-time 0.50s wps 11.16K ppl 16.34 bleu 0.00'\n",
      "b'  global step 2900 lr 1 step-time 0.50s wps 11.05K ppl 15.54 bleu 0.00'\n",
      "b'  global step 3000 lr 1 step-time 0.51s wps 11.20K ppl 15.60 bleu 0.00'\n",
      "b'# Save eval, global step 3000'\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-3000\n",
      "b'  loaded infer model parameters from /data/manni/mt_model/translate.ckpt-3000, time 0.13s'\n",
      "b'  # 1193'\n",
      "    src: Now how did he come to be looking for help in this very unique manner ?\n",
      "    ref: Làm thế nào nó có thể tìm kiếm sự giúp đỡ trong tình trạng có một không hai như thế này ?\n",
      "    nmt: Bây giờ anh ấy đi tìm kiếm sự giúp đỡ trong một cách độc đáo ? </s> <unk> <unk>\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-3000\n",
      "b'  loaded eval model parameters from /data/manni/mt_model/translate.ckpt-3000, time 0.19s'\n",
      "  eval dev: perplexity 18.15, time 3s, Thu Jul 27 20:01:00 2017.\n",
      "b'  global step 3100 lr 1 step-time 0.50s wps 11.15K ppl 15.35 bleu 0.00'\n",
      "b'# Finished an epoch, step 3129. Perform external evaluation'\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-3000\n",
      "b'  loaded infer model parameters from /data/manni/mt_model/translate.ckpt-3000, time 0.22s'\n",
      "b'  # 957'\n",
      "    src: And so I jumped in , I jumped into this world of technology , to see how I could use it to enable magic as opposed to kill it .\n",
      "    ref: Và thế là tôi nhảy vào , nhảy vào thế giới công nghệ này , để xem tôi có thể dùng nó để tạo ra phép màu như thế nào chứ không phải là tiêu diệt nó .\n",
      "    nmt: Và vì vậy tôi nhảy vào đó , tôi nhảy vào thế giới này , để xem tôi có thể sử dụng nó để có thể đối mặt với nó . </s> <unk> <unk>\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-3000\n",
      "b'  loaded infer model parameters from /data/manni/mt_model/translate.ckpt-3000, time 0.22s'\n",
      "b'# External evaluation, global step 3000'\n",
      "b'  decoding to output /data/manni/mt_model/output_dev.'\n",
      "  done, num sentences 1553, time 69s, Thu Jul 27 20:03:15 2017.\n",
      "b'  bleu dev: 0.0'\n",
      "b'  saving hparams to /data/manni/mt_model/hparams'\n",
      "b'  global step 3200 lr 1 step-time 0.57s wps 9.56K ppl 13.35 bleu 0.00'\n",
      "b'  global step 3300 lr 1 step-time 0.52s wps 10.90K ppl 12.67 bleu 0.00'\n",
      "b'  global step 3400 lr 1 step-time 0.50s wps 11.19K ppl 12.61 bleu 0.00'\n",
      "b'  global step 3500 lr 1 step-time 0.50s wps 11.18K ppl 12.41 bleu 0.00'\n",
      "b'  global step 3600 lr 1 step-time 0.50s wps 11.07K ppl 12.14 bleu 0.00'\n",
      "b'  global step 3700 lr 1 step-time 0.50s wps 11.13K ppl 12.04 bleu 0.00'\n",
      "b'  global step 3800 lr 1 step-time 0.51s wps 11.16K ppl 12.25 bleu 0.00'\n",
      "b'  global step 3900 lr 1 step-time 0.51s wps 11.09K ppl 11.91 bleu 0.00'\n",
      "b'  global step 4000 lr 1 step-time 0.50s wps 11.13K ppl 11.77 bleu 0.00'\n",
      "b'# Save eval, global step 4000'\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-4000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "b'  loaded infer model parameters from /data/manni/mt_model/translate.ckpt-4000, time 0.23s'\n",
      "b'  # 1358'\n",
      "    src: The other thing we would like to ask is of companies also all over the world that will be able to help us validate these AEDs .\n",
      "    ref: Một điều khác nữa chúng tôi xin rất mong được giúp đỡ là các công ty trên khắp thế giới có thể giúp chúng tôi duy trì các thiết bị AED này .\n",
      "    nmt: Một điều khác mà chúng tôi muốn đặt ra là các công ty cũng có thể giúp chúng ta có thể giúp chúng ta vượt qua những điều đó . </s> <unk> <unk> <unk> <unk> <unk> <unk> <unk>\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-4000\n",
      "b'  loaded eval model parameters from /data/manni/mt_model/translate.ckpt-4000, time 0.20s'\n",
      "  eval dev: perplexity 14.41, time 3s, Thu Jul 27 20:10:50 2017.\n",
      "b'  global step 4100 lr 1 step-time 0.50s wps 11.09K ppl 11.58 bleu 0.00'\n",
      "b'# Finished an epoch, step 4172. Perform external evaluation'\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-4000\n",
      "b'  loaded infer model parameters from /data/manni/mt_model/translate.ckpt-4000, time 0.21s'\n",
      "b'  # 520'\n",
      "    src: So when I ask you a question , since I &apos;m blind , only raise your hand if you want to burn off some calories .\n",
      "    ref: Cho nên khi tôi hỏi bạn 1 câu hỏi Cho là tôi không nhìn thấy gì , nếu bạn muốn đốt 1 lượng calo , chỉ cần giơ tay cao\n",
      "    nmt: Vì vậy khi tôi hỏi các bạn một câu hỏi , vì tôi bị mù , chỉ giơ tay nếu bạn muốn đốt tắt . </s> </s> <unk> <unk> <unk> <unk>\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-4000\n",
      "b'  loaded infer model parameters from /data/manni/mt_model/translate.ckpt-4000, time 0.21s'\n",
      "b'# External evaluation, global step 4000'\n",
      "b'  decoding to output /data/manni/mt_model/output_dev.'\n",
      "  done, num sentences 1553, time 91s, Thu Jul 27 20:13:49 2017.\n",
      "b'  bleu dev: 0.1'\n",
      "b'  saving hparams to /data/manni/mt_model/hparams'\n",
      "b'  global step 4200 lr 1 step-time 0.54s wps 10.38K ppl 10.88 bleu 0.09'\n",
      "b'  global step 4300 lr 1 step-time 0.56s wps 10.17K ppl 9.89 bleu 0.09'\n",
      "b'  global step 4400 lr 1 step-time 0.51s wps 11.14K ppl 9.85 bleu 0.09'\n",
      "b'  global step 4500 lr 1 step-time 0.50s wps 11.06K ppl 9.80 bleu 0.09'\n",
      "b'  global step 4600 lr 1 step-time 0.50s wps 11.18K ppl 9.77 bleu 0.09'\n",
      "b'  global step 4700 lr 1 step-time 0.51s wps 11.18K ppl 9.98 bleu 0.09'\n",
      "b'  global step 4800 lr 1 step-time 0.50s wps 11.09K ppl 9.82 bleu 0.09'\n",
      "b'  global step 4900 lr 1 step-time 0.51s wps 11.08K ppl 9.81 bleu 0.09'\n",
      "b'  global step 5000 lr 1 step-time 0.50s wps 11.15K ppl 9.71 bleu 0.09'\n",
      "b'# Save eval, global step 5000'\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-5000\n",
      "b'  loaded infer model parameters from /data/manni/mt_model/translate.ckpt-5000, time 0.13s'\n",
      "b'  # 1088'\n",
      "    src: And in fact , we &apos;re such a multi-cellular community .\n",
      "    ref: Và trong thực tế , chúng ta chính là một quần thể đa tế bào .\n",
      "    nmt: Và thực tế , chúng tôi là một cộng đồng <unk> . </s> <unk>\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-5000\n",
      "b'  loaded eval model parameters from /data/manni/mt_model/translate.ckpt-5000, time 0.21s'\n",
      "  eval dev: perplexity 13.93, time 3s, Thu Jul 27 20:21:04 2017.\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-5000\n",
      "b'  loaded infer model parameters from /data/manni/mt_model/translate.ckpt-5000, time 0.16s'\n",
      "b'  # 1042'\n",
      "    src: Now here &apos;s a conference in which people talk about the future , and you notice that the future is still at about the year 2000 .\n",
      "    ref: Đây là một hội nghị mà ở đó mọi người bàn về tương lai , và mọi người có thể nhận thấy tương lai đó vẫn xoay quanh năm 2000 .\n",
      "    nmt: Đây là một hội nghị mà mọi người nói về tương lai , và bạn thấy rằng tương lai vẫn ở năm 2000 . </s> <unk> <unk> <unk> <unk>\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-5000\n",
      "b'  loaded infer model parameters from /data/manni/mt_model/translate.ckpt-5000, time 0.24s'\n",
      "b'# External evaluation, global step 5000'\n",
      "b'  decoding to output /data/manni/mt_model/output_dev.'\n",
      "  done, num sentences 1553, time 81s, Thu Jul 27 20:22:32 2017.\n",
      "b'  bleu dev: 0.1'\n",
      "b'  saving hparams to /data/manni/mt_model/hparams'\n",
      "b'  global step 5100 lr 1 step-time 0.51s wps 11.21K ppl 9.73 bleu 0.15'\n",
      "b'  global step 5200 lr 1 step-time 0.50s wps 11.13K ppl 9.56 bleu 0.15'\n",
      "b'# Finished an epoch, step 5215. Perform external evaluation'\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-5000\n",
      "b'  loaded infer model parameters from /data/manni/mt_model/translate.ckpt-5000, time 0.21s'\n",
      "b'  # 593'\n",
      "    src: Now in fact , you had objectively more information the first time around than the second time around , but I would venture to guess that you felt that it was more real the second time around .\n",
      "    ref: Thưc tế thì Bạn có nhiều thông tin khách quan Lần đâu tiên sau đó lần thứ hai nhưng tôi sẽ đoán thử rằng bạn cảm thấy nó thực hơn trong lần thứ hai\n",
      "    nmt: Thực tế , các bạn đã biết thêm thông tin lần đầu tiên vòng quanh , nhưng tôi sẽ cho rằng bạn cảm thấy rằng nó có thực sự là lần thứ hai . </s> <unk> <unk> <unk> <unk> <unk>\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-5000\n",
      "b'  loaded infer model parameters from /data/manni/mt_model/translate.ckpt-5000, time 0.20s'\n",
      "b'# External evaluation, global step 5000'\n",
      "b'  decoding to output /data/manni/mt_model/output_dev.'\n",
      "  done, num sentences 1553, time 81s, Thu Jul 27 20:25:46 2017.\n",
      "b'  bleu dev: 0.1'\n",
      "b'  saving hparams to /data/manni/mt_model/hparams'\n",
      "b'  global step 5300 lr 1 step-time 0.58s wps 9.46K ppl 8.27 bleu 0.15'\n",
      "b'  global step 5400 lr 1 step-time 0.51s wps 11.09K ppl 8.21 bleu 0.15'\n",
      "b'  global step 5500 lr 1 step-time 0.50s wps 11.16K ppl 8.33 bleu 0.15'\n",
      "b'  global step 5600 lr 1 step-time 0.51s wps 11.14K ppl 8.41 bleu 0.15'\n",
      "b'  global step 5700 lr 1 step-time 0.51s wps 11.18K ppl 8.48 bleu 0.15'\n",
      "b'  global step 5800 lr 1 step-time 0.50s wps 11.12K ppl 8.48 bleu 0.15'\n",
      "b'  global step 5900 lr 1 step-time 0.50s wps 11.18K ppl 8.45 bleu 0.15'\n",
      "b'  global step 6000 lr 1 step-time 0.51s wps 11.24K ppl 8.62 bleu 0.15'\n",
      "b'# Save eval, global step 6000'\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-6000\n",
      "b'  loaded infer model parameters from /data/manni/mt_model/translate.ckpt-6000, time 0.21s'\n",
      "b'  # 818'\n",
      "    src: We work in a system where errors happen every day , where one in 10 medications are either the wrong medication given in hospital or at the wrong dosage , where hospital-acquired infections are getting more and more numerous , causing havoc and death .\n",
      "    ref: Ta làm việc trong một hệ thống trong đó sai sót xảy ra hàng ngày , trong đó cứ 10 đơn thuốc thì có 1 đơn hoặc là kê nhầm thuốc hoặc là kê nhầm liều lượng , trong đó các ca nhiễm khuẩn bắt nguồn từ bệnh viện ngày càng nhiều , dẫn tới tàn phá cơ thể và chết chóc .\n",
      "    nmt: Chúng tôi làm việc trong 1 hệ thống nơi mà sai lầm xảy ra mỗi ngày , nơi mà một trong số 10 phương thuốc đều được đưa ra thuốc trong bệnh viện hoặc ở <unk> <unk> , nơi mà sự lây nhiễm bị nhiễm </s> <unk> <unk> <unk>\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-6000\n",
      "b'  loaded eval model parameters from /data/manni/mt_model/translate.ckpt-6000, time 0.20s'\n",
      "  eval dev: perplexity 12.65, time 3s, Thu Jul 27 20:32:37 2017.\n",
      "b'  global step 6100 lr 1 step-time 0.50s wps 11.11K ppl 8.42 bleu 0.15'\n",
      "b'  global step 6200 lr 1 step-time 0.50s wps 11.14K ppl 8.51 bleu 0.15'\n",
      "b'# Finished an epoch, step 6258. Perform external evaluation'\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-6000\n",
      "b'  loaded infer model parameters from /data/manni/mt_model/translate.ckpt-6000, time 0.21s'\n",
      "b'  # 173'\n",
      "    src: And we were kind of just out there doing the stuff we crazily did .\n",
      "    ref: đại loại là chúng tôi đã ở ngoài đó làm những thứ điên cuồng\n",
      "    nmt: Và chúng tôi chỉ ra ngoài đó . </s> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk> <unk>\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-6000\n",
      "b'  loaded infer model parameters from /data/manni/mt_model/translate.ckpt-6000, time 0.24s'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "b'# External evaluation, global step 6000'\n",
      "b'  decoding to output /data/manni/mt_model/output_dev.'\n",
      "  done, num sentences 1553, time 82s, Thu Jul 27 20:36:10 2017.\n",
      "b'  bleu dev: 0.1'\n",
      "b'  saving hparams to /data/manni/mt_model/hparams'\n",
      "b'  global step 6300 lr 1 step-time 0.55s wps 10.04K ppl 7.84 bleu 0.15'\n",
      "b'  global step 6400 lr 1 step-time 0.54s wps 10.57K ppl 7.23 bleu 0.15'\n",
      "b'  global step 6500 lr 1 step-time 0.50s wps 11.07K ppl 7.30 bleu 0.15'\n",
      "b'  global step 6600 lr 1 step-time 0.50s wps 11.18K ppl 7.49 bleu 0.15'\n",
      "b'  global step 6700 lr 1 step-time 0.51s wps 11.19K ppl 7.50 bleu 0.15'\n",
      "b'  global step 6800 lr 1 step-time 0.50s wps 11.13K ppl 7.48 bleu 0.15'\n",
      "b'  global step 6900 lr 1 step-time 0.51s wps 10.98K ppl 7.62 bleu 0.15'\n",
      "b'  global step 7000 lr 1 step-time 0.51s wps 11.08K ppl 7.74 bleu 0.15'\n",
      "b'# Save eval, global step 7000'\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-7000\n",
      "b'  loaded infer model parameters from /data/manni/mt_model/translate.ckpt-7000, time 0.35s'\n",
      "b'  # 1149'\n",
      "    src: Take two programs and they produce children by exchanging their subroutines , and the children inherit the traits of the subroutines of the two programs .\n",
      "    ref: Lấy ra hai chương trình và chúng sẽ tạo ra sản phẩm bằng cách trao đổi quy tắc ngầm của chúng , và sản phẩm được thừa hưởng tính chất của các quy tắc của hai chương trình .\n",
      "    nmt: Hãy xem xét hai chương trình và sản sinh trẻ em bằng cách trao đổi con cá nhân của mình , và những đứa trẻ thừa nhận đặc tính của hai phần mềm . </s> <unk> <unk>\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-7000\n",
      "b'  loaded eval model parameters from /data/manni/mt_model/translate.ckpt-7000, time 0.20s'\n",
      "  eval dev: perplexity 12.11, time 3s, Thu Jul 27 20:42:41 2017.\n",
      "b'  global step 7100 lr 1 step-time 0.50s wps 11.01K ppl 7.50 bleu 0.15'\n",
      "b'  global step 7200 lr 1 step-time 0.51s wps 11.11K ppl 7.67 bleu 0.15'\n",
      "b'  global step 7300 lr 1 step-time 0.50s wps 10.86K ppl 7.64 bleu 0.15'\n",
      "b'# Finished an epoch, step 7301. Perform external evaluation'\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-7000\n",
      "b'  loaded infer model parameters from /data/manni/mt_model/translate.ckpt-7000, time 0.20s'\n",
      "b'  # 992'\n",
      "    src: When I was a child , I had multiple collections of sticks and stones and pebbles and shells .\n",
      "    ref: Khi còn nhỏ , tôi có hàng loạt bộ sưu tập que , đá , sỏi và vỏ sò .\n",
      "    nmt: Khi còn là một đứa trẻ , tôi đã có nhiều bộ sưu tập gỗ và đá và những cái vỏ . </s> <unk> <unk>\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-7000\n",
      "b'  loaded infer model parameters from /data/manni/mt_model/translate.ckpt-7000, time 0.23s'\n",
      "b'# External evaluation, global step 7000'\n",
      "b'  decoding to output /data/manni/mt_model/output_dev.'\n",
      "  done, num sentences 1553, time 97s, Thu Jul 27 20:46:52 2017.\n",
      "b'  bleu dev: 0.2'\n",
      "b'  saving hparams to /data/manni/mt_model/hparams'\n",
      "b'  global step 7400 lr 1 step-time 0.58s wps 9.68K ppl 6.56 bleu 0.16'\n",
      "b'  global step 7500 lr 1 step-time 0.51s wps 11.08K ppl 6.66 bleu 0.16'\n",
      "b'  global step 7600 lr 1 step-time 0.51s wps 11.14K ppl 6.70 bleu 0.16'\n",
      "b'  global step 7700 lr 1 step-time 0.50s wps 11.15K ppl 6.84 bleu 0.16'\n",
      "b'  global step 7800 lr 1 step-time 0.51s wps 11.09K ppl 6.83 bleu 0.16'\n",
      "b'  global step 7900 lr 1 step-time 0.51s wps 11.13K ppl 6.91 bleu 0.16'\n",
      "b'  global step 8000 lr 1 step-time 0.51s wps 11.18K ppl 6.90 bleu 0.16'\n",
      "b'# Save eval, global step 8000'\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-8000\n",
      "b'  loaded infer model parameters from /data/manni/mt_model/translate.ckpt-8000, time 0.15s'\n",
      "b'  # 255'\n",
      "    src: We &apos;re constantly running into each other .\n",
      "    ref: chúng ta không ngừng mâu thuẫn\n",
      "    nmt: Chúng ta luôn chạy vào nhau . </s> <unk>\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-8000\n",
      "b'  loaded eval model parameters from /data/manni/mt_model/translate.ckpt-8000, time 0.20s'\n",
      "  eval dev: perplexity 12.33, time 3s, Thu Jul 27 20:53:02 2017.\n",
      "b'  global step 8100 lr 1 step-time 0.51s wps 11.09K ppl 6.95 bleu 0.16'\n",
      "b'  global step 8200 lr 1 step-time 0.50s wps 11.07K ppl 7.06 bleu 0.16'\n",
      "b'  global step 8300 lr 1 step-time 0.51s wps 11.16K ppl 7.15 bleu 0.16'\n",
      "b'# Finished an epoch, step 8344. Perform external evaluation'\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-8000\n",
      "b'  loaded infer model parameters from /data/manni/mt_model/translate.ckpt-8000, time 0.20s'\n",
      "b'  # 242'\n",
      "    src: And it &apos;s interesting , when I teach my students about African-American history , I tell them about slavery .\n",
      "    ref: Và điều thú vị là khi tôi dạy học sinh của mình về lịch sử của người Mỹ gốc Phi Tôi kể với họ về chế độ nô lệ ,\n",
      "    nmt: Và thú vị là , khi tôi dạy những học sinh của mình về lịch sử Mỹ gốc Phi , tôi nói về chế độ nô lệ . </s> <unk> <unk> <unk>\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-8000\n",
      "b'  loaded infer model parameters from /data/manni/mt_model/translate.ckpt-8000, time 0.21s'\n",
      "b'# External evaluation, global step 8000'\n",
      "b'  decoding to output /data/manni/mt_model/output_dev.'\n",
      "  done, num sentences 1553, time 85s, Thu Jul 27 20:57:22 2017.\n",
      "b'  bleu dev: 0.1'\n",
      "b'  saving hparams to /data/manni/mt_model/hparams'\n",
      "b'  global step 8400 lr 1 step-time 0.56s wps 9.83K ppl 6.41 bleu 0.16'\n",
      "b'  global step 8500 lr 1 step-time 0.52s wps 10.83K ppl 6.11 bleu 0.16'\n",
      "b'  global step 8600 lr 1 step-time 0.51s wps 11.19K ppl 6.26 bleu 0.16'\n",
      "b'  global step 8700 lr 1 step-time 0.51s wps 11.11K ppl 6.19 bleu 0.16'\n",
      "b'  global step 8800 lr 1 step-time 0.50s wps 11.18K ppl 6.32 bleu 0.16'\n",
      "b'  global step 8900 lr 1 step-time 0.50s wps 11.18K ppl 6.30 bleu 0.16'\n",
      "b'  global step 9000 lr 0.5 step-time 0.50s wps 11.16K ppl 6.38 bleu 0.16'\n",
      "b'# Save eval, global step 9000'\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-9000\n",
      "b'  loaded infer model parameters from /data/manni/mt_model/translate.ckpt-9000, time 0.29s'\n",
      "b'  # 491'\n",
      "    src: So you have a situation where people don &apos;t know the past , even though we live in literate societies , because they don &apos;t trust the sources of the past .\n",
      "    ref: Vậy nên bạn có một tình huống trong đó mọi người không biết đến quá khứ thậm chí ngay cả khi họ đang sống trong xã hội chữ nghĩa , vì họ không tin tưởng những gì đã tồn tại quá khứ\n",
      "    nmt: Vì thế bạn có tình huống mà người ta không biết quá khứ , mặc dù chúng ta sống trong xã hội <unk> , bởi vì họ không tin vào các nguồn tin của quá khứ . </s> <unk>\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-9000\n",
      "b'  loaded eval model parameters from /data/manni/mt_model/translate.ckpt-9000, time 0.20s'\n",
      "  eval dev: perplexity 11.74, time 2s, Thu Jul 27 21:03:06 2017.\n",
      "b'  global step 9100 lr 0.5 step-time 0.51s wps 11.17K ppl 6.19 bleu 0.16'\n",
      "b'  global step 9200 lr 0.5 step-time 0.50s wps 11.12K ppl 5.87 bleu 0.16'\n",
      "b'  global step 9300 lr 0.5 step-time 0.50s wps 11.17K ppl 5.86 bleu 0.16'\n",
      "b'# Finished an epoch, step 9387. Perform external evaluation'\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-9000\n",
      "b'  loaded infer model parameters from /data/manni/mt_model/translate.ckpt-9000, time 0.26s'\n",
      "b'  # 1273'\n",
      "    src: It connected those two people .\n",
      "    ref: Nó kết nối hai người đó lại với nhau .\n",
      "    nmt: Nó kết nối hai người này . </s> <unk> <unk>\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-9000\n",
      "b'  loaded infer model parameters from /data/manni/mt_model/translate.ckpt-9000, time 0.26s'\n",
      "b'# External evaluation, global step 9000'\n",
      "b'  decoding to output /data/manni/mt_model/output_dev.'\n",
      "  done, num sentences 1553, time 88s, Thu Jul 27 21:07:50 2017.\n",
      "b'  bleu dev: 0.1'\n",
      "b'  saving hparams to /data/manni/mt_model/hparams'\n",
      "b'  global step 9400 lr 0.5 step-time 0.52s wps 10.58K ppl 5.73 bleu 0.16'\n",
      "b'  global step 9500 lr 0.5 step-time 0.56s wps 10.12K ppl 4.98 bleu 0.16'\n",
      "b'  global step 9600 lr 0.5 step-time 0.50s wps 11.18K ppl 5.02 bleu 0.16'\n",
      "b'  global step 9700 lr 0.5 step-time 0.50s wps 11.09K ppl 5.05 bleu 0.16'\n",
      "b'  global step 9800 lr 0.5 step-time 0.50s wps 11.14K ppl 5.14 bleu 0.16'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "b'  global step 9900 lr 0.5 step-time 0.51s wps 11.10K ppl 5.16 bleu 0.16'\n",
      "b'  global step 10000 lr 0.25 step-time 0.51s wps 11.18K ppl 5.26 bleu 0.16'\n",
      "b'# Save eval, global step 10000'\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-10000\n",
      "b'  loaded infer model parameters from /data/manni/mt_model/translate.ckpt-10000, time 0.12s'\n",
      "b'  # 1268'\n",
      "    src: And it &apos;s also a great example of government getting in on the crowd-sourcing game .\n",
      "    ref: Đây cũng là một ví dụ điển hình khác về việc chính phủ triển khai trò chơi cộng đồng .\n",
      "    nmt: Và nó cũng là một ví dụ tuyệt vời về chính phủ đang tham gia vào trò chơi <unk> . </s>\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-10000\n",
      "b'  loaded eval model parameters from /data/manni/mt_model/translate.ckpt-10000, time 0.32s'\n",
      "  eval dev: perplexity 11.80, time 3s, Thu Jul 27 21:13:14 2017.\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-10000\n",
      "b'  loaded infer model parameters from /data/manni/mt_model/translate.ckpt-10000, time 0.23s'\n",
      "b'  # 9'\n",
      "    src: He died broken by history .\n",
      "    ref: Ông qua đời , bị lịch sử quật ngã .\n",
      "    nmt: Ông chết bởi lịch sử . </s> <unk>\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-10000\n",
      "b'  loaded infer model parameters from /data/manni/mt_model/translate.ckpt-10000, time 0.23s'\n",
      "b'# External evaluation, global step 10000'\n",
      "b'  decoding to output /data/manni/mt_model/output_dev.'\n",
      "  done, num sentences 1553, time 86s, Thu Jul 27 21:14:47 2017.\n",
      "b'  bleu dev: 0.2'\n",
      "b'  saving hparams to /data/manni/mt_model/hparams'\n",
      "b'  global step 10100 lr 0.25 step-time 0.50s wps 11.19K ppl 5.12 bleu 0.23'\n",
      "b'  global step 10200 lr 0.25 step-time 0.50s wps 11.07K ppl 5.03 bleu 0.23'\n",
      "b'  global step 10300 lr 0.25 step-time 0.51s wps 11.08K ppl 5.03 bleu 0.23'\n",
      "b'  global step 10400 lr 0.25 step-time 0.50s wps 11.13K ppl 4.99 bleu 0.23'\n",
      "b'# Finished an epoch, step 10430. Perform external evaluation'\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-10000\n",
      "b'  loaded infer model parameters from /data/manni/mt_model/translate.ckpt-10000, time 0.21s'\n",
      "b'  # 1536'\n",
      "    src: But Western governments are doing it to themselves as well .\n",
      "    ref: Nhưng chính phủ phương Tây cũng thực hiện điều đó ngay tại quốc gia của mình\n",
      "    nmt: Nhưng các nước phương Tây đang làm điều đó cho chính mình . </s> <unk> <unk>\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-10000\n",
      "b'  loaded infer model parameters from /data/manni/mt_model/translate.ckpt-10000, time 0.20s'\n",
      "b'# External evaluation, global step 10000'\n",
      "b'  decoding to output /data/manni/mt_model/output_dev.'\n",
      "  done, num sentences 1553, time 86s, Thu Jul 27 21:19:53 2017.\n",
      "b'  bleu dev: 0.2'\n",
      "b'  saving hparams to /data/manni/mt_model/hparams'\n",
      "b'  global step 10500 lr 0.25 step-time 0.56s wps 9.64K ppl 4.62 bleu 0.23'\n",
      "b'  global step 10600 lr 0.25 step-time 0.51s wps 11.05K ppl 4.54 bleu 0.23'\n",
      "b'  global step 10700 lr 0.25 step-time 0.51s wps 11.13K ppl 4.54 bleu 0.23'\n",
      "b'  global step 10800 lr 0.25 step-time 0.50s wps 11.12K ppl 4.52 bleu 0.23'\n",
      "b'  global step 10900 lr 0.25 step-time 0.52s wps 11.14K ppl 4.64 bleu 0.23'\n",
      "b'  global step 11000 lr 0.125 step-time 0.50s wps 11.18K ppl 4.59 bleu 0.23'\n",
      "b'# Save eval, global step 11000'\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-11000\n",
      "b'  loaded infer model parameters from /data/manni/mt_model/translate.ckpt-11000, time 0.14s'\n",
      "b'  # 1462'\n",
      "    src: And it &apos;s been an incredible experience .\n",
      "    ref: Và đó là một trải nghiệm tuyệt vời .\n",
      "    nmt: Đó là một trải nghiệm đáng kinh ngạc . </s> <unk>\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-11000\n",
      "b'  loaded eval model parameters from /data/manni/mt_model/translate.ckpt-11000, time 0.20s'\n",
      "  eval dev: perplexity 11.68, time 3s, Thu Jul 27 21:24:55 2017.\n",
      "b'  global step 11100 lr 0.125 step-time 0.50s wps 11.11K ppl 4.54 bleu 0.23'\n",
      "b'  global step 11200 lr 0.125 step-time 0.51s wps 11.23K ppl 4.60 bleu 0.23'\n",
      "b'  global step 11300 lr 0.125 step-time 0.50s wps 11.13K ppl 4.55 bleu 0.23'\n",
      "b'  global step 11400 lr 0.125 step-time 0.50s wps 11.25K ppl 4.60 bleu 0.23'\n",
      "b'# Finished an epoch, step 11473. Perform external evaluation'\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-11000\n",
      "b'  loaded infer model parameters from /data/manni/mt_model/translate.ckpt-11000, time 0.26s'\n",
      "b'  # 124'\n",
      "    src: And I mention that because I think identity is really important .\n",
      "    ref: và tôi đề cập đến điều này bởi tôi nghĩ bản sắc thực sự rất quan trọng\n",
      "    nmt: Và tôi đề cập vì tôi nghĩ rằng danh tính thực sự quan trọng . </s> <unk> <unk> <unk>\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-11000\n",
      "b'  loaded infer model parameters from /data/manni/mt_model/translate.ckpt-11000, time 0.23s'\n",
      "b'# External evaluation, global step 11000'\n",
      "b'  decoding to output /data/manni/mt_model/output_dev.'\n",
      "  done, num sentences 1553, time 86s, Thu Jul 27 21:30:21 2017.\n",
      "b'  bleu dev: 0.2'\n",
      "b'  saving hparams to /data/manni/mt_model/hparams'\n",
      "b'  global step 11500 lr 0.125 step-time 0.53s wps 10.32K ppl 4.43 bleu 0.23'\n",
      "b'  global step 11600 lr 0.125 step-time 0.54s wps 10.36K ppl 4.27 bleu 0.23'\n",
      "b'  global step 11700 lr 0.125 step-time 0.51s wps 11.24K ppl 4.27 bleu 0.23'\n",
      "b'  global step 11800 lr 0.125 step-time 0.50s wps 11.17K ppl 4.29 bleu 0.23'\n",
      "b'  global step 11900 lr 0.125 step-time 0.50s wps 11.05K ppl 4.31 bleu 0.23'\n",
      "b'  global step 12000 lr 0.0625 step-time 0.51s wps 11.16K ppl 4.36 bleu 0.23'\n",
      "b'# Save eval, global step 12000'\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-12000\n",
      "b'  loaded infer model parameters from /data/manni/mt_model/translate.ckpt-12000, time 0.21s'\n",
      "b'  # 846'\n",
      "    src: So the system is evolving to create backups that make it easier to detect those mistakes that humans inevitably make and also fosters in a loving , supportive way places where everybody who is observing in the health care system can actually point out things that could be potential mistakes and is rewarded for doing so , and especially people like me , when we do make mistakes , we &apos;re rewarded for coming clean .\n",
      "    ref: Nên hệ thống ấy đang tiến triển để tạo ra phương án dự phòng để việc phát hiện sai lầm dễ dàng hơn những sai lầm mà con người không thể không mắc và cúng tạo ra , một cách tận tình , ủng hộ , những nơi mà bất kì ai theo dõi trong hệ thống chăm sóc sức khoẻ đều có thể chỉ ra những điều có thể trở thành sai lầm và được thưởng khi làm việc đó , và đặc biệt là những người như tôi , khi chúng tôi phạm sai lầm thật , chúng tôi được thưởng vì nói trắng ra điều đó .\n",
      "    nmt: Do đó , hệ thống đang tiến hoá để tạo ra những sai lầm làm cho nó dễ dàng phát hiện ra những sai lầm mà con người không thể tránh được và trở thành những sai lầm mà con người không thể biết chắc chắn là những sai lầm mà con người có thể quan tâm ra khỏi những sai lầm chăm sóc sức khoẻ có thể chỉ ra rằng có thể là sai lầm </s>\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-12000\n",
      "b'  loaded eval model parameters from /data/manni/mt_model/translate.ckpt-12000, time 0.20s'\n",
      "  eval dev: perplexity 11.97, time 3s, Thu Jul 27 21:35:01 2017.\n"
     ]
    }
   ],
   "source": [
    "import math\n",
    "\n",
    "while global_step < hparams.num_train_steps:\n",
    "    start_time = time.time()\n",
    "    try:\n",
    "        step_result = loaded_train_model.train(train_sess)\n",
    "        (_, step_loss, step_predict_count, step_summary, global_step, step_word_count, \n",
    "        batch_size) = step_result\n",
    "        hparams.epoch_step += 1\n",
    "    except tf.errors.OutOfRangeError:\n",
    "        #Next Epoch\n",
    "        hparams.epoch_step = 0\n",
    "        utils.print_out(\"# Finished an epoch, step %d. Perform external evaluation\" %global_step)\n",
    "        Train.run_sample_decode(infer_graph, \n",
    "                          infer_sess, \n",
    "                          hparams.out_dir, \n",
    "                          hparams, \n",
    "                          summary_writer, \n",
    "                          sample_src_data,\n",
    "                          sample_tgt_data)\n",
    "        dev_scores, test_scores, _ = Train.run_external_eval(infer_graph, \n",
    "                                                             infer_sess, \n",
    "                                                             hparams.out_dir,\n",
    "                                                             hparams, \n",
    "                                                             summary_writer)\n",
    "        train_sess.run(train_graph.iterator.initializer,feed_dict={train_graph.skip_count_placeholder: 0})\n",
    "        continue\n",
    "\n",
    "    summary_writer.add_summary(step_summary, global_step)\n",
    "    \n",
    "    #Statistics\n",
    "    step_time += (time.time() - start_time)\n",
    "    checkpoint_loss += (step_loss * batch_size)\n",
    "    checkpoint_predict_count += step_predict_count\n",
    "    checkpoint_total_count += float(step_word_count)\n",
    "\n",
    "    #print statistics\n",
    "    if global_step - last_stats_step >= hparams.steps_per_stats:\n",
    "        last_stats_step = global_step\n",
    "        avg_step_time = step_time / hparams.steps_per_stats\n",
    "        train_ppl = utils.safe_exp(checkpoint_loss / checkpoint_predict_count)\n",
    "        speed = checkpoint_total_count / (1000 * step_time)\n",
    "        \n",
    "        utils.print_out(\"  global step %d lr %g \"\n",
    "          \"step-time %.2fs wps %.2fK ppl %.2f %s\" %\n",
    "          (global_step,\n",
    "           loaded_train_model.learning_rate.eval(session=train_sess),\n",
    "           avg_step_time, speed, train_ppl, Train._get_best_results(hparams)))\n",
    "        \n",
    "        if math.isnan(train_ppl):\n",
    "            break\n",
    "\n",
    "        # Reset timer and loss.\n",
    "        step_time, checkpoint_loss, checkpoint_predict_count = 0.0, 0.0, 0.0\n",
    "        checkpoint_total_count = 0.0\n",
    "        \n",
    "\n",
    "    if global_step - last_eval_step >= steps_per_eval:\n",
    "        last_eval_step = global_step\n",
    "        utils.print_out(\"# Save eval, global step %d\" % global_step)\n",
    "        utils.add_summary(summary_writer, global_step, \"train_ppl\", train_ppl)\n",
    "\n",
    "        # Save checkpoint\n",
    "        loaded_train_model.saver.save(train_sess,os.path.join(hparams.out_dir, \"translate.ckpt\"),\n",
    "                                      global_step=global_step)\n",
    "\n",
    "        # Evaluate on dev/test\n",
    "        Train.run_sample_decode(infer_graph, \n",
    "                                infer_sess, \n",
    "                                out_dir, \n",
    "                                hparams, \n",
    "                                summary_writer, \n",
    "                                sample_src_data,\n",
    "                                sample_tgt_data)\n",
    "        dev_ppl, test_ppl = Train.run_internal_eval(eval_graph, \n",
    "                                                    eval_sess, \n",
    "                                                    out_dir, \n",
    "                                                    hparams, \n",
    "                                                    summary_writer)\n",
    "\n",
    "    if global_step - last_external_eval_step >= steps_per_external_eval:\n",
    "        last_external_eval_step = global_step\n",
    "        \n",
    "        # Save checkpoint\n",
    "        loaded_train_model.saver.save(train_sess,os.path.join(hparams.out_dir, \"translate.ckpt\"),\n",
    "                                      global_step=global_step)\n",
    "        \n",
    "        Train.run_sample_decode(infer_graph, \n",
    "                                infer_sess,\n",
    "                                out_dir, \n",
    "                                hparams, \n",
    "                                summary_writer, \n",
    "                                sample_src_data,\n",
    "                                sample_tgt_data)\n",
    "        dev_scores, test_scores, _ = Train.run_external_eval(infer_graph, \n",
    "                                                             infer_sess, \n",
    "                                                             out_dir,\n",
    "                                                             hparams, \n",
    "                                                             summary_writer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-27T20:35:06.748995Z",
     "start_time": "2017-07-27T20:35:02.025859Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/data/manni/mt_model/translate.ckpt-12000'"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Save training\n",
    "loaded_train_model.saver.save(train_sess,os.path.join(out_dir, \"translate.ckpt\"),global_step=global_step)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-27T20:38:10.025126Z",
     "start_time": "2017-07-27T20:35:07.122477Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-12000\n",
      "b'  loaded infer model parameters from /data/manni/mt_model/translate.ckpt-12000, time 0.20s'\n",
      "b'  # 94'\n",
      "    src: With no support from her husband , she caused a sensation by taking him to court and prosecuting her own case , and a far greater sensation when she won .\n",
      "    ref: Và dầu vấp phải sự không ủng hộ của chồng , bà gây chấn động khi kiện ông ta ra toà và tự khởi tố , và còn gây chấn động mạnh hơn khi bà thắng kiện .\n",
      "    nmt: Không có sự ủng hộ từ chồng cô ấy , cô đã tạo ra một cảm giác bằng cách đưa ông đến phiên toà và làm cho chính trường hợp của cô ấy một cảm giác lớn hơn khi cô ấy thắng . </s> </s> <unk>\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-12000\n",
      "b'  loaded eval model parameters from /data/manni/mt_model/translate.ckpt-12000, time 0.20s'\n",
      "  eval dev: perplexity 11.97, time 3s, Thu Jul 27 21:35:11 2017.\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-12000\n",
      "b'  loaded infer model parameters from /data/manni/mt_model/translate.ckpt-12000, time 0.21s'\n",
      "b'# External evaluation, global step 12000'\n",
      "b'  decoding to output /data/manni/mt_model/output_dev.'\n",
      "  done, num sentences 1553, time 86s, Thu Jul 27 21:36:38 2017.\n",
      "b'  bleu dev: 0.2'\n",
      "b'  saving hparams to /data/manni/mt_model/hparams'\n",
      "b'# Final, step 12000 lr 0.0625 step-time 0.51 wps 11.16K ppl 4.36, dev ppl 11.97, dev bleu 0.2, Thu Jul 27 21:36:38 2017'\n",
      "# Done training!, time 7482s, Thu Jul 27 21:36:38 2017.\n",
      "b'# Start evaluating saved best models.'\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/best_bleu/translate.ckpt-10000\n",
      "b'  loaded infer model parameters from /data/manni/mt_model/best_bleu/translate.ckpt-10000, time 0.24s'\n",
      "b'  # 923'\n",
      "    src: And we all now feel some ownership in our own pop culture .\n",
      "    ref: Và tất cả chúng ta đều cảm thấy vài sự sở hữu trong nền văn hoá nhạc pop của chính mình .\n",
      "    nmt: Và tất cả chúng ta đều cảm thấy quyền sở hữu trong nền văn hoá pop của mình . </s> <unk> <unk> <unk>\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/best_bleu/translate.ckpt-10000\n",
      "b'  loaded eval model parameters from /data/manni/mt_model/best_bleu/translate.ckpt-10000, time 0.23s'\n",
      "  eval dev: perplexity 11.80, time 3s, Thu Jul 27 21:36:42 2017.\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/best_bleu/translate.ckpt-10000\n",
      "b'  loaded infer model parameters from /data/manni/mt_model/best_bleu/translate.ckpt-10000, time 0.23s'\n",
      "b'# External evaluation, global step 10000'\n",
      "b'  decoding to output /data/manni/mt_model/output_dev.'\n",
      "  done, num sentences 1553, time 86s, Thu Jul 27 21:38:09 2017.\n",
      "b'  bleu dev: 0.2'\n",
      "b'  saving hparams to /data/manni/mt_model/hparams'\n",
      "b'# Best bleu, step 10000 step-time 0.51 wps 11.16K, dev ppl 11.80, dev bleu 0.2, Thu Jul 27 21:38:10 2017'\n"
     ]
    }
   ],
   "source": [
    "import evaluation_utils\n",
    "\n",
    "eval_results, _ = Train.run_full_eval(out_dir, \n",
    "                                infer_graph, \n",
    "                                infer_sess,\n",
    "                                eval_graph, \n",
    "                                eval_sess, \n",
    "                                hparams,\n",
    "                                summary_writer, \n",
    "                                sample_src_data,\n",
    "                                sample_tgt_data)\n",
    "\n",
    "##Printing stats\n",
    "utils.print_out(\"# Final, step %d lr %g \"\n",
    "                \"step-time %.2f wps %.2fK ppl %.2f, %s, %s\" %\n",
    "                (global_step, loaded_train_model.learning_rate.eval(session=train_sess),\n",
    "                avg_step_time, speed, train_ppl, eval_results, time.ctime()))\n",
    "utils.print_time(\"# Done training!\", start_train_time)\n",
    "\n",
    "utils.print_out(\"# Start evaluating saved best models.\")\n",
    "\n",
    "for metric in hparams.metrics:\n",
    "    best_model_dir = getattr(hparams, \"best_\" + metric + \"_dir\")\n",
    "    eval_results, best_global_step = Train.run_full_eval(best_model_dir, \n",
    "                                                         infer_graph, \n",
    "                                                         infer_sess, \n",
    "                                                         eval_graph, \n",
    "                                                         eval_sess, \n",
    "                                                         hparams,\n",
    "                                                         summary_writer, \n",
    "                                                         sample_src_data, \n",
    "                                                         sample_tgt_data)\n",
    "    utils.print_out(\"# Best %s, step %d \"\n",
    "                    \"step-time %.2f wps %.2fK, %s, %s\" %\n",
    "                    (metric, best_global_step, avg_step_time, speed,eval_results, time.ctime()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Inference / Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-28T07:59:50.920417Z",
     "start_time": "2017-07-28T07:59:50.891889Z"
    }
   },
   "outputs": [],
   "source": [
    "hparams.add_hparam(\"inference_input_file\",\"/data/manni/mt_data/tst2013.en\")\n",
    "hparams.add_hparam(\"inference_output_file\",\"/data/manni/mt_data/tst2013_pred.vi\")\n",
    "hparams.add_hparam(\"inference_ref_file\",\"/data/manni/mt_data/tst2013.vi\")\n",
    "hparams.add_hparam(\"jobid\",0)\n",
    "hparams.add_hparam(\"num_workers\",1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-28T08:03:43.098816Z",
     "start_time": "2017-07-28T08:02:19.207134Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "b'# creating infer graph ...'\n",
      "b'  num_bi_layers = 1, num_bi_residual_layers=0'\n",
      "b'  cell 0'b'  LSTM, forget_bias=1'b'  DeviceWrapper, device=/gpu:0'b''\n",
      "b'  cell 0'b'  LSTM, forget_bias=1'b'  DeviceWrapper, device=/gpu:0'b''\n",
      "b'  cell 0'b'  LSTM, forget_bias=1'b'  DeviceWrapper, device=/gpu:0'b''\n",
      "b'  cell 1'b'  LSTM, forget_bias=1'b'  DeviceWrapper, device=/gpu:0'b''\n",
      "  start_decay_step=8000, learning_rate=1, decay_steps 1000,decay_factor 0.5\n",
      "b'# Trainable variables'\n",
      "b'  embeddings/encoder/embedding_encoder:0, (17191, 512), '\n",
      "b'  embeddings/decoder/embedding_decoder:0, (7709, 512), '\n",
      "b'  dynamic_seq2seq/encoder/bidirectional_rnn/fw/basic_lstm_cell/kernel:0, (1024, 2048), /device:GPU:0'\n",
      "b'  dynamic_seq2seq/encoder/bidirectional_rnn/fw/basic_lstm_cell/bias:0, (2048,), /device:GPU:0'\n",
      "b'  dynamic_seq2seq/encoder/bidirectional_rnn/bw/basic_lstm_cell/kernel:0, (1024, 2048), /device:GPU:0'\n",
      "b'  dynamic_seq2seq/encoder/bidirectional_rnn/bw/basic_lstm_cell/bias:0, (2048,), /device:GPU:0'\n",
      "b'  dynamic_seq2seq/decoder/memory_layer/kernel:0, (1024, 512), '\n",
      "b'  dynamic_seq2seq/decoder/attention/multi_rnn_cell/cell_0/basic_lstm_cell/kernel:0, (1536, 2048), /device:GPU:0'\n",
      "b'  dynamic_seq2seq/decoder/attention/multi_rnn_cell/cell_0/basic_lstm_cell/bias:0, (2048,), /device:GPU:0'\n",
      "b'  dynamic_seq2seq/decoder/attention/multi_rnn_cell/cell_1/basic_lstm_cell/kernel:0, (1024, 2048), /device:GPU:0'\n",
      "b'  dynamic_seq2seq/decoder/attention/multi_rnn_cell/cell_1/basic_lstm_cell/bias:0, (2048,), /device:GPU:0'\n",
      "b'  dynamic_seq2seq/decoder/attention/luong_attention/attention_g:0, (), /device:GPU:0'\n",
      "b'  dynamic_seq2seq/decoder/attention/attention_layer/kernel:0, (1536, 512), /device:GPU:0'\n",
      "b'  dynamic_seq2seq/decoder/output_projection/kernel:0, (512, 7709), '\n",
      "INFO:tensorflow:Restoring parameters from /data/manni/mt_model/translate.ckpt-12000\n",
      "b'  loaded infer model parameters from /data/manni/mt_model/translate.ckpt-12000, time 0.64s'\n",
      "b'# Start decoding'\n",
      "b'  decoding to output /data/manni/mt_data/tst2013_pred.vi.'\n",
      "  done, num sentences 1268, time 80s, Fri Jul 28 09:03:42 2017.\n",
      "b'  bleu: 0.3'\n"
     ]
    }
   ],
   "source": [
    "import inference\n",
    "\n",
    "trans_file = hparams.inference_output_file\n",
    "ckpt = hparams.ckpt\n",
    "if not ckpt:\n",
    "    ckpt = tf.train.latest_checkpoint(out_dir)\n",
    "    inference.inference(ckpt, hparams.inference_input_file, trans_file, hparams, hparams.num_workers, hparams.jobid)\n",
    "    ref_file = hparams.inference_ref_file\n",
    "    if ref_file and tf.gfile.Exists(trans_file):\n",
    "        for metric in hparams.metrics:\n",
    "            score = evaluation_utils.evaluate(ref_file,\n",
    "                                              trans_file,\n",
    "                                              metric,\n",
    "                                              hparams.bpe_delimiter)\n",
    "            utils.print_out(\"  %s: %.1f\" % (metric, score))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}