one_million/all-word/Model-aw-lex-local_attention-fast-v2-6.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### fast local attention + hidden states  \n",
    "#### learning rate = 0.001,  windows size 5, batch_size = 128, num_gpus = 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "tf.logging.set_verbosity(tf.logging.WARN)\n",
    "import pickle\n",
    "import numpy as np\n",
    "import os\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import accuracy_score\n",
    "import os\n",
    "from tensorflow.python.client import device_lib\n",
    "from collections import Counter\n",
    "import time\n",
    "\n",
    "VERY_BIG_NUMBER = 1e30"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = open('../../Glove/word_embedding_glove', 'rb')\n",
    "word_embedding = pickle.load(f)\n",
    "f.close()\n",
    "\n",
    "word_embedding = word_embedding[: len(word_embedding)-1]\n",
    "\n",
    "f = open('../../Glove/vocab_glove', 'rb')\n",
    "vocab = pickle.load(f)\n",
    "f.close()\n",
    "\n",
    "word2id = dict((w, i) for i,w in enumerate(vocab))\n",
    "id2word = dict((i, w) for i,w in enumerate(vocab))\n",
    "\n",
    "unknown_token = \"UNKNOWN_TOKEN\"\n",
    "\n",
    "# Model Description\n",
    "model_name = 'model-aw-lex-local-att-fast-v2-6'\n",
    "model_dir = '../output/all-word/' + model_name\n",
    "save_dir = os.path.join(model_dir, \"save/\")\n",
    "log_dir = os.path.join(model_dir, \"log\")\n",
    "\n",
    "if not os.path.exists(model_dir):\n",
    "    os.mkdir(model_dir)\n",
    "if not os.path.exists(save_dir):\n",
    "    os.mkdir(save_dir)\n",
    "if not os.path.exists(log_dir):\n",
    "    os.mkdir(log_dir)\n",
    "\n",
    "with open('../../../dataset/train_val_data_fine/all_word_lex','rb') as f:\n",
    "    train_data, val_data = pickle.load(f)    \n",
    "    \n",
    "# Parameters\n",
    "mode = 'train'\n",
    "num_senses = 45\n",
    "num_pos = 12\n",
    "batch_size = 128\n",
    "vocab_size = len(vocab)\n",
    "unk_vocab_size = 1\n",
    "word_emb_size = len(word_embedding[0])\n",
    "max_sent_size = 200\n",
    "hidden_size = 256\n",
    "num_filter = 256\n",
    "window_size = 5\n",
    "kernel_size = 5\n",
    "keep_prob = 0.3\n",
    "l2_lambda = 0.001\n",
    "init_lr = 0.001\n",
    "decay_steps = 500\n",
    "decay_rate = 0.99\n",
    "clip_norm = 1\n",
    "clipping = True\n",
    "moving_avg_deacy = 0.999\n",
    "num_gpus = 6\n",
    "width = int(window_size/2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def average_gradients(tower_grads):\n",
    "    average_grads = []\n",
    "    for grad_and_vars in zip(*tower_grads):\n",
    "        # Note that each grad_and_vars looks like the following:\n",
    "        #   ((grad0_gpu0, var0_gpu0), ... , (grad0_gpuN, var0_gpuN))\n",
    "        grads = []\n",
    "        for g, _ in grad_and_vars:\n",
    "            # Add 0 dimension to the gradients to represent the tower.\n",
    "            expanded_g = tf.expand_dims(g, 0)\n",
    "\n",
    "            # Append on a 'tower' dimension which we will average over below.\n",
    "            grads.append(expanded_g)\n",
    "\n",
    "        # Average over the 'tower' dimension.\n",
    "        grad = tf.concat(grads, 0)\n",
    "        grad = tf.reduce_mean(grad, axis=0)\n",
    "\n",
    "        # Keep in mind that the Variables are redundant because they are shared\n",
    "        # across towers. So .. we will just return the first tower's pointer to\n",
    "        # the Variable.\n",
    "        v = grad_and_vars[0][1]\n",
    "        grad_and_var = (grad, v)\n",
    "        average_grads.append(grad_and_var)\n",
    "    return average_grads"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/sshanukr/env/lib/python3.5/site-packages/tensorflow/python/ops/clip_ops.py:113: calling reduce_sum (from tensorflow.python.ops.math_ops) with keep_dims is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "keep_dims is deprecated, use keepdims instead\n"
     ]
    }
   ],
   "source": [
    "# MODEL\n",
    "device_num = 0\n",
    "tower_grads = []\n",
    "losses = []\n",
    "predictions = []\n",
    "predictions_pos = []\n",
    "\n",
    "x = tf.placeholder('int32', [num_gpus, batch_size, max_sent_size], name=\"x\")\n",
    "y = tf.placeholder('int32', [num_gpus, batch_size, max_sent_size], name=\"y\")\n",
    "y_pos = tf.placeholder('int32', [num_gpus, batch_size, max_sent_size], name=\"y\")\n",
    "x_mask  = tf.placeholder('bool', [num_gpus, batch_size, max_sent_size], name='x_mask') \n",
    "sense_mask  = tf.placeholder('bool', [num_gpus, batch_size, max_sent_size], name='sense_mask')\n",
    "is_train = tf.placeholder('bool', [], name='is_train')\n",
    "word_emb_mat = tf.placeholder('float', [None, word_emb_size], name='emb_mat')\n",
    "input_keep_prob = tf.cond(is_train,lambda:keep_prob, lambda:tf.constant(1.0))\n",
    "pretrain = tf.placeholder('bool', [], name=\"pretrain\")\n",
    "\n",
    "global_step = tf.Variable(0, trainable=False, name=\"global_step\")\n",
    "learning_rate = tf.train.exponential_decay(init_lr, global_step, decay_steps, decay_rate, staircase=True)\n",
    "summaries = []\n",
    "\n",
    "def global_attention(input_x, input_mask, W_att):\n",
    "    h_masked = tf.boolean_mask(input_x, input_mask)\n",
    "    h_tanh = tf.tanh(h_masked)\n",
    "    u = tf.matmul(h_tanh, W_att)\n",
    "    a = tf.nn.softmax(u)\n",
    "    c = tf.reduce_sum(tf.multiply(h_masked, a), axis=0)  \n",
    "    return c\n",
    "\n",
    "with tf.variable_scope(\"word_embedding\"):\n",
    "    unk_word_emb_mat = tf.get_variable(\"word_emb_mat\", dtype='float', shape=[unk_vocab_size, word_emb_size], initializer=tf.contrib.layers.xavier_initializer(uniform=True, seed=0, dtype=tf.float32))\n",
    "    final_word_emb_mat = tf.concat([word_emb_mat, unk_word_emb_mat], 0)\n",
    "\n",
    "with tf.variable_scope(tf.get_variable_scope()):\n",
    "    for gpu_idx in range(num_gpus):\n",
    "        if gpu_idx>=3:\n",
    "            device_num = 1\n",
    "        with tf.name_scope(\"model_{}\".format(gpu_idx)) as scope, tf.device('/gpu:%d' % device_num):\n",
    "\n",
    "            if gpu_idx > 0:\n",
    "                    tf.get_variable_scope().reuse_variables()\n",
    "\n",
    "            with tf.name_scope(\"word\"):\n",
    "                Wx = tf.nn.embedding_lookup(final_word_emb_mat, x[gpu_idx])  \n",
    "\n",
    "            float_x_mask = tf.cast(x_mask[gpu_idx], 'float')\n",
    "            x_len = tf.reduce_sum(tf.cast(x_mask[gpu_idx], 'int32'), axis=1)\n",
    "            \n",
    "            with tf.variable_scope(\"lstm1\"):\n",
    "                cell_fw1 = tf.contrib.rnn.BasicLSTMCell(hidden_size,state_is_tuple=True)\n",
    "                cell_bw1 = tf.contrib.rnn.BasicLSTMCell(hidden_size,state_is_tuple=True)\n",
    "\n",
    "                d_cell_fw1 = tf.contrib.rnn.DropoutWrapper(cell_fw1, input_keep_prob=input_keep_prob)\n",
    "                d_cell_bw1 = tf.contrib.rnn.DropoutWrapper(cell_bw1, input_keep_prob=input_keep_prob)\n",
    "\n",
    "                (fw_h1, bw_h1), _ = tf.nn.bidirectional_dynamic_rnn(d_cell_fw1, d_cell_bw1, Wx, sequence_length=x_len, dtype='float', scope='lstm1')\n",
    "                h1 = tf.concat([fw_h1, bw_h1], 2)\n",
    "\n",
    "            with tf.variable_scope(\"lstm2\"):\n",
    "                cell_fw2 = tf.contrib.rnn.BasicLSTMCell(hidden_size,state_is_tuple=True)\n",
    "                cell_bw2 = tf.contrib.rnn.BasicLSTMCell(hidden_size,state_is_tuple=True)\n",
    "\n",
    "                d_cell_fw2 = tf.contrib.rnn.DropoutWrapper(cell_fw2, input_keep_prob=input_keep_prob)\n",
    "                d_cell_bw2 = tf.contrib.rnn.DropoutWrapper(cell_bw2, input_keep_prob=input_keep_prob)\n",
    "\n",
    "                (fw_h2, bw_h2), _ = tf.nn.bidirectional_dynamic_rnn(d_cell_fw2, d_cell_bw2, h1, sequence_length=x_len, dtype='float', scope='lstm2')\n",
    "                h = tf.concat([fw_h2, bw_h2], 2)\n",
    "            \n",
    "            with tf.variable_scope(\"local_attention\"):\n",
    "                W_att_local = tf.get_variable(\"W_att_local\", shape=[2*hidden_size, 1], initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1, seed=gpu_idx*10))\n",
    "                flat_h = tf.reshape(h, [batch_size*max_sent_size, tf.shape(h)[2]])\n",
    "                h_tanh = tf.tanh(flat_h)\n",
    "                u_flat = tf.matmul(h_tanh, W_att_local)\n",
    "                u_local = tf.reshape(u_flat, [batch_size, max_sent_size])\n",
    "                final_u = (tf.cast(x_mask[gpu_idx], 'float') -1)*VERY_BIG_NUMBER + u_local\n",
    "                c_local = tf.map_fn(lambda i:tf.reduce_sum(tf.multiply(h[:, tf.maximum(0, i-width-1):tf.minimum(1+width+i, max_sent_size)],\n",
    "                    tf.expand_dims(tf.nn.softmax(final_u[:, tf.maximum(0, i-width-1):tf.minimum(1+width+i, max_sent_size)], 1), 2)), axis=1),\n",
    "                    tf.range(max_sent_size), dtype=tf.float32)  \n",
    "            \n",
    "            c_local = tf.transpose(c_local, perm=[1,0,2]) \n",
    "            h_final = tf.concat([tf.multiply(c_local, tf.expand_dims(float_x_mask, 2)), h], 2)\n",
    "            flat_h_final = tf.reshape(h_final, [-1, tf.shape(h_final)[2]])\n",
    "            with tf.variable_scope(\"hidden_layer\"):\n",
    "                W = tf.get_variable(\"W\", shape=[4*hidden_size, 2*hidden_size], initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1, seed=gpu_idx*20))\n",
    "                b = tf.get_variable(\"b\", shape=[2*hidden_size], initializer=tf.zeros_initializer())\n",
    "                drop_flat_h_final = tf.nn.dropout(flat_h_final, input_keep_prob)\n",
    "                flat_hl = tf.matmul(drop_flat_h_final, W) + b\n",
    "\n",
    "            with tf.variable_scope(\"softmax_layer\"):\n",
    "                W = tf.get_variable(\"W\", shape=[2*hidden_size, num_senses], initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1, seed=gpu_idx*20))\n",
    "                b = tf.get_variable(\"b\", shape=[num_senses], initializer=tf.zeros_initializer())\n",
    "                drop_flat_hl = tf.nn.dropout(flat_hl, input_keep_prob)\n",
    "                flat_logits_sense = tf.matmul(drop_flat_hl, W) + b\n",
    "                logits = tf.reshape(flat_logits_sense, [batch_size, max_sent_size, num_senses])\n",
    "                predictions.append(tf.argmax(logits, 2))\n",
    "\n",
    "            with tf.variable_scope(\"softmax_layer_pos\"):\n",
    "                W = tf.get_variable(\"W\", shape=[2*hidden_size, num_pos], initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1, seed=gpu_idx*30))\n",
    "                b = tf.get_variable(\"b\", shape=[num_pos], initializer=tf.zeros_initializer())\n",
    "                flat_h1 = tf.reshape(h1, [-1, tf.shape(h1)[2]])\n",
    "                drop_flat_hl = tf.nn.dropout(flat_hl, input_keep_prob)\n",
    "                flat_logits_pos = tf.matmul(drop_flat_hl, W) + b\n",
    "                logits_pos = tf.reshape(flat_logits_pos, [batch_size, max_sent_size, num_pos])\n",
    "                predictions_pos.append(tf.argmax(logits_pos, 2))\n",
    "\n",
    "\n",
    "            float_sense_mask = tf.cast(sense_mask[gpu_idx], 'float')\n",
    "\n",
    "            loss = tf.contrib.seq2seq.sequence_loss(logits, y[gpu_idx], float_sense_mask, name=\"loss\")\n",
    "            loss_pos = tf.contrib.seq2seq.sequence_loss(logits_pos, y_pos[gpu_idx], float_x_mask, name=\"loss_\")\n",
    "\n",
    "            l2_loss = l2_lambda * tf.losses.get_regularization_loss()\n",
    "\n",
    "            total_loss = tf.cond(pretrain, lambda:loss_pos, lambda:loss + loss_pos + l2_loss)\n",
    "\n",
    "            summaries.append(tf.summary.scalar(\"loss_{}\".format(gpu_idx), loss))\n",
    "            summaries.append(tf.summary.scalar(\"loss_pos_{}\".format(gpu_idx), loss_pos))\n",
    "            summaries.append(tf.summary.scalar(\"total_loss_{}\".format(gpu_idx), total_loss))\n",
    "\n",
    "\n",
    "            optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "            grads_vars = optimizer.compute_gradients(total_loss)\n",
    "\n",
    "            clipped_grads = grads_vars\n",
    "            if(clipping == True):\n",
    "                clipped_grads = [(tf.clip_by_norm(grad, clip_norm), var) for grad, var in clipped_grads]\n",
    "\n",
    "            tower_grads.append(clipped_grads)\n",
    "            losses.append(total_loss)\n",
    "\n",
    "with tf.device('/gpu:0'):\n",
    "    tower_grads = average_gradients(tower_grads)\n",
    "    losses = tf.add_n(losses)/len(losses)\n",
    "    apply_grad_op = optimizer.apply_gradients(tower_grads, global_step=global_step)\n",
    "    summaries.append(tf.summary.scalar('total_loss', losses))\n",
    "    summaries.append(tf.summary.scalar('learning_rate', learning_rate))\n",
    "\n",
    "    variable_averages = tf.train.ExponentialMovingAverage(moving_avg_deacy, global_step)\n",
    "    variables_averages_op = variable_averages.apply(tf.trainable_variables())\n",
    "\n",
    "    train_op = tf.group(apply_grad_op, variables_averages_op)\n",
    "    saver = tf.train.Saver(tf.global_variables())\n",
    "    summary = tf.summary.merge(summaries)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.environ[\"CUDA_DEVICE_ORDER\"]=\"PCI_BUS_ID\"   # see issue #152\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"1,2\"\n",
    "# print (device_lib.list_local_devices())\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth = True\n",
    "config.allow_soft_placement = True\n",
    "sess = tf.Session(config=config)\n",
    "sess.run(tf.global_variables_initializer())                          # For initializing all the variables\n",
    "summary_writer = tf.summary.FileWriter(log_dir, sess.graph)          # For writing Summaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_period = 100\n",
    "log_period = 100\n",
    "\n",
    "def model(xx, yy, yy_pos, mask, smask, train_cond=True, pretrain_cond=False):\n",
    "    num_batches = int(len(xx)/(batch_size*num_gpus))\n",
    "    _losses = 0\n",
    "    temp_loss = 0\n",
    "    preds_sense = []\n",
    "    true_sense = []\n",
    "    preds_pos = []\n",
    "    true_pos = []\n",
    "    \n",
    "    for j in range(num_batches): \n",
    "        \n",
    "        s = j * batch_size * num_gpus\n",
    "        e = (j+1) * batch_size * num_gpus\n",
    "        xx_re = xx[s:e].reshape([num_gpus, batch_size, -1])\n",
    "        yy_re = yy[s:e].reshape([num_gpus, batch_size, -1])\n",
    "        yy_pos_re = yy_pos[s:e].reshape([num_gpus, batch_size, -1])\n",
    "        mask_re = mask[s:e].reshape([num_gpus, batch_size, -1])\n",
    "        smask_re = smask[s:e].reshape([num_gpus, batch_size, -1])\n",
    " \n",
    "        feed_dict = {x:xx_re, y:yy_re, y_pos:yy_pos_re, x_mask:mask_re, sense_mask:smask_re, pretrain:pretrain_cond, is_train:train_cond, input_keep_prob:keep_prob, word_emb_mat:word_embedding}\n",
    "        \n",
    "        if(train_cond==True):\n",
    "            _, _loss, step, _summary = sess.run([train_op, losses, global_step, summary], feed_dict)\n",
    "            summary_writer.add_summary(_summary, step)\n",
    "            \n",
    "            temp_loss += _loss\n",
    "            if((j+1)%log_period==0):\n",
    "                print(\"Steps: {}\".format(step), \"Loss:{0:.4f}\".format(temp_loss/log_period), \", Current Loss: {0:.4f}\".format(_loss))\n",
    "                temp_loss = 0\n",
    "            if((j+1)%save_period==0):\n",
    "                saver.save(sess, save_path=save_dir)                         \n",
    "                \n",
    "        else:\n",
    "            _loss, pred, pred_pos = sess.run([total_loss, predictions, predictions_pos], feed_dict)\n",
    "            for i in range(num_gpus):\n",
    "                preds_sense.append(pred[i][smask_re[i]])\n",
    "                true_sense.append(yy_re[i][smask_re[i]])\n",
    "                preds_pos.append(pred_pos[i][mask_re[i]])\n",
    "                true_pos.append(yy_pos_re[i][mask_re[i]])\n",
    "\n",
    "        _losses +=_loss\n",
    "\n",
    "    if(train_cond==False): \n",
    "        sense_preds = []\n",
    "        sense_true = []\n",
    "        pos_preds = []\n",
    "        pos_true = []\n",
    "        \n",
    "        for preds in preds_sense:\n",
    "            for ps in preds:      \n",
    "                sense_preds.append(ps)  \n",
    "        for trues in true_sense:\n",
    "            for ts in trues:\n",
    "                sense_true.append(ts)\n",
    "        \n",
    "        for preds in preds_pos:\n",
    "            for ps in preds:      \n",
    "                pos_preds.append(ps)      \n",
    "        for trues in true_pos:\n",
    "            for ts in trues:\n",
    "                pos_true.append(ts)\n",
    "                \n",
    "        return _losses/num_batches, sense_preds, sense_true, pos_preds, pos_true\n",
    "\n",
    "    return _losses/num_batches, step\n",
    "\n",
    "def eval_score(yy, pred, yy_pos, pred_pos):\n",
    "    f1 = f1_score(yy, pred, average='macro')\n",
    "    accu = accuracy_score(yy, pred)\n",
    "    f1_pos = f1_score(yy_pos, pred_pos, average='macro')\n",
    "    accu_pos = accuracy_score(yy_pos, pred_pos)\n",
    "    return f1*100, accu*100, f1_pos*100, accu_pos*100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_id_train = train_data['x']\n",
    "mask_train = train_data['x_mask']\n",
    "sense_mask_train = train_data['sense_mask']\n",
    "y_train = train_data['y']\n",
    "y_pos_train = train_data['pos']\n",
    "\n",
    "x_id_val = val_data['x']\n",
    "mask_val = val_data['x_mask']\n",
    "sense_mask_val = val_data['sense_mask']\n",
    "y_val = val_data['y']\n",
    "y_pos_val = val_data['pos']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def testing():\n",
    "    start_time = time.time()\n",
    "    val_loss, val_pred, val_true, val_pred_pos, val_true_pos = model(x_id_val, y_val, y_pos_val, mask_val, sense_mask_val, train_cond=False)        \n",
    "    f1_, accu_, f1_pos_, accu_pos_ = eval_score(val_true, val_pred, val_true_pos, val_pred_pos)\n",
    "    time_taken = time.time() - start_time\n",
    "    print(\"Val: F1 Score:{0:.2f}\".format(f1_), \"Accuracy:{0:.2f}\".format(accu_), \" POS: F1 Score:{0:.2f}\".format(f1_pos_), \"Accuracy:{0:.2f}\".format(accu_pos_), \"Loss:{0:.4f}\".format(val_loss), \", Time: {0:.1f}\".format(time_taken))\n",
    "    return f1_, accu_, f1_pos_, accu_pos_\n",
    "\n",
    "def training(current_epoch, pre_train_cond):\n",
    "        random = np.random.choice(len(y_train), size=(len(y_train)), replace=False)\n",
    "        x_id_train_tmp = x_id_train[random]\n",
    "        y_train_tmp = y_train[random]\n",
    "        mask_train_tmp = mask_train[random]    \n",
    "        sense_mask_train_tmp = sense_mask_train[random]\n",
    "        y_pos_train_tmp = y_pos_train[random]\n",
    "\n",
    "        start_time = time.time()\n",
    "        train_loss, step = model(x_id_train_tmp, y_train_tmp, y_pos_train_tmp, mask_train_tmp, sense_mask_train_tmp, pretrain_cond=pre_train_cond)\n",
    "        time_taken = time.time() - start_time\n",
    "        print(\"Epoch: {}\".format(current_epoch+1),\", Step: {}\".format(step), \", loss: {0:.4f}\".format(train_loss), \", Time: {0:.1f}\".format(time_taken))\n",
    "        saver.save(sess, save_path=save_dir)                         \n",
    "        print(\"Model Saved\")\n",
    "        return [step, train_loss]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Steps: 100 Loss:1.6015 , Current Loss: 0.7926\n",
      "Steps: 200 Loss:0.6486 , Current Loss: 0.5507\n",
      "Steps: 300 Loss:0.5023 , Current Loss: 0.4660\n",
      "Steps: 400 Loss:0.4269 , Current Loss: 0.4007\n",
      "Steps: 500 Loss:0.3825 , Current Loss: 0.3656\n",
      "Steps: 600 Loss:0.3523 , Current Loss: 0.3421\n",
      "Steps: 700 Loss:0.3307 , Current Loss: 0.3227\n",
      "Steps: 800 Loss:0.3131 , Current Loss: 0.3048\n",
      "Epoch: 1 , Step: 885 , loss: 0.5437 , Time: 7279.5\n",
      "Model Saved\n",
      "Steps: 985 Loss:0.2893 , Current Loss: 0.2879\n",
      "Steps: 1085 Loss:0.2777 , Current Loss: 0.2692\n",
      "Steps: 1185 Loss:0.2696 , Current Loss: 0.2671\n",
      "Steps: 1285 Loss:0.2620 , Current Loss: 0.2507\n",
      "Steps: 1385 Loss:0.2539 , Current Loss: 0.2451\n",
      "Steps: 1485 Loss:0.2480 , Current Loss: 0.2395\n",
      "Steps: 1585 Loss:0.2440 , Current Loss: 0.2575\n",
      "Steps: 1685 Loss:0.2375 , Current Loss: 0.2359\n",
      "Epoch: 2 , Step: 1770 , loss: 0.2576 , Time: 6970.7\n",
      "Model Saved\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/sshanukr/env/lib/python3.5/site-packages/sklearn/metrics/classification.py:1137: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val: F1 Score:1.35 Accuracy:1.95  POS: F1 Score:86.70 Accuracy:92.12 Loss:6.2910 , Time: 1410.4\n",
      "Steps: 1870 Loss:2.8840 , Current Loss: 2.5844\n",
      "Steps: 1970 Loss:2.5170 , Current Loss: 2.5246\n",
      "Steps: 2070 Loss:2.4178 , Current Loss: 2.3329\n",
      "Steps: 2170 Loss:2.3540 , Current Loss: 2.3197\n",
      "Steps: 2270 Loss:2.3206 , Current Loss: 2.2919\n",
      "Steps: 2370 Loss:2.2744 , Current Loss: 2.2894\n",
      "Steps: 2470 Loss:2.2626 , Current Loss: 2.2247\n",
      "Steps: 2570 Loss:2.2309 , Current Loss: 2.1419\n",
      "Epoch: 1 , Step: 2655 , loss: 2.3892 , Time: 6812.4\n",
      "Model Saved\n",
      "Steps: 2755 Loss:2.1892 , Current Loss: 2.2220\n",
      "Steps: 2855 Loss:2.1744 , Current Loss: 2.1280\n",
      "Steps: 2955 Loss:2.1623 , Current Loss: 2.1958\n",
      "Steps: 3055 Loss:2.1407 , Current Loss: 2.1413\n",
      "Steps: 3155 Loss:2.1300 , Current Loss: 2.1237\n",
      "Steps: 3255 Loss:2.1234 , Current Loss: 2.1278\n",
      "Steps: 3355 Loss:2.1098 , Current Loss: 2.1013\n",
      "Steps: 3455 Loss:2.0993 , Current Loss: 2.0977\n",
      "Epoch: 2 , Step: 3540 , loss: 2.1355 , Time: 6806.5\n",
      "Model Saved\n",
      "Val: F1 Score:31.64 Accuracy:44.15  POS: F1 Score:82.45 Accuracy:90.59 Loss:2.0905 , Time: 1411.7\n",
      "Steps: 3640 Loss:2.0688 , Current Loss: 2.1838\n",
      "Steps: 3740 Loss:2.0588 , Current Loss: 2.0763\n",
      "Steps: 3840 Loss:2.0658 , Current Loss: 2.0792\n",
      "Steps: 3940 Loss:2.0498 , Current Loss: 2.0158\n",
      "Steps: 4040 Loss:2.0550 , Current Loss: 2.0131\n",
      "Steps: 4140 Loss:2.0370 , Current Loss: 2.0824\n",
      "Steps: 4240 Loss:2.0308 , Current Loss: 1.9560\n",
      "Steps: 4340 Loss:2.0156 , Current Loss: 2.1157\n",
      "Epoch: 3 , Step: 4425 , loss: 2.0456 , Time: 6806.4\n",
      "Model Saved\n",
      "Steps: 4525 Loss:2.0026 , Current Loss: 2.0495\n",
      "Steps: 4625 Loss:1.9989 , Current Loss: 1.9991\n",
      "Steps: 4725 Loss:2.0028 , Current Loss: 2.0050\n",
      "Steps: 4825 Loss:1.9951 , Current Loss: 1.9621\n",
      "Steps: 4925 Loss:1.9946 , Current Loss: 2.0460\n",
      "Steps: 5025 Loss:1.9748 , Current Loss: 1.9060\n",
      "Steps: 5125 Loss:1.9818 , Current Loss: 2.0524\n",
      "Steps: 5225 Loss:1.9682 , Current Loss: 1.9255\n",
      "Epoch: 4 , Step: 5310 , loss: 1.9871 , Time: 6796.3\n",
      "Model Saved\n",
      "Val: F1 Score:36.35 Accuracy:47.12  POS: F1 Score:82.84 Accuracy:90.92 Loss:1.9670 , Time: 1409.6\n",
      "Steps: 5410 Loss:1.9634 , Current Loss: 1.9474\n",
      "Steps: 5510 Loss:1.9531 , Current Loss: 1.9126\n",
      "Steps: 5610 Loss:1.9506 , Current Loss: 1.9189\n",
      "Steps: 5710 Loss:1.9441 , Current Loss: 2.0294\n",
      "Steps: 5810 Loss:1.9415 , Current Loss: 1.9593\n",
      "Steps: 5910 Loss:1.9292 , Current Loss: 1.9854\n",
      "Steps: 6010 Loss:1.9341 , Current Loss: 1.9370\n",
      "Steps: 6110 Loss:1.9350 , Current Loss: 2.0254\n",
      "Epoch: 5 , Step: 6195 , loss: 1.9412 , Time: 6802.8\n",
      "Model Saved\n",
      "Steps: 6295 Loss:1.9251 , Current Loss: 1.9187\n",
      "Steps: 6395 Loss:1.9140 , Current Loss: 1.9163\n",
      "Steps: 6495 Loss:1.9076 , Current Loss: 1.8694\n",
      "Steps: 6595 Loss:1.9133 , Current Loss: 1.9754\n",
      "Steps: 6695 Loss:1.9096 , Current Loss: 1.9788\n",
      "Steps: 6795 Loss:1.9081 , Current Loss: 1.9092\n",
      "Steps: 6895 Loss:1.8947 , Current Loss: 1.9162\n",
      "Steps: 6995 Loss:1.8928 , Current Loss: 1.9878\n",
      "Epoch: 6 , Step: 7080 , loss: 1.9064 , Time: 6807.1\n",
      "Model Saved\n",
      "Val: F1 Score:38.25 Accuracy:48.92  POS: F1 Score:83.21 Accuracy:91.16 Loss:1.9108 , Time: 1415.4\n",
      "Steps: 7180 Loss:1.8838 , Current Loss: 1.8849\n",
      "Steps: 7280 Loss:1.8823 , Current Loss: 1.9388\n",
      "Steps: 7380 Loss:1.8835 , Current Loss: 1.8659\n",
      "Steps: 7480 Loss:1.8888 , Current Loss: 1.8564\n",
      "Steps: 7580 Loss:1.8786 , Current Loss: 1.8623\n",
      "Steps: 7680 Loss:1.8817 , Current Loss: 1.8487\n",
      "Steps: 7780 Loss:1.8763 , Current Loss: 1.8169\n",
      "Steps: 7880 Loss:1.8631 , Current Loss: 1.8783\n",
      "Epoch: 7 , Step: 7965 , loss: 1.8779 , Time: 6840.4\n",
      "Model Saved\n",
      "Steps: 8065 Loss:1.8623 , Current Loss: 1.8601\n",
      "Steps: 8165 Loss:1.8630 , Current Loss: 1.8248\n",
      "Steps: 8265 Loss:1.8526 , Current Loss: 1.8367\n",
      "Steps: 8365 Loss:1.8619 , Current Loss: 1.8145\n",
      "Steps: 8465 Loss:1.8593 , Current Loss: 1.9002\n",
      "Steps: 8565 Loss:1.8545 , Current Loss: 1.8449\n",
      "Steps: 8665 Loss:1.8451 , Current Loss: 1.8406\n",
      "Steps: 8765 Loss:1.8449 , Current Loss: 1.8544\n",
      "Epoch: 8 , Step: 8850 , loss: 1.8544 , Time: 6806.7\n",
      "Model Saved\n",
      "Val: F1 Score:39.96 Accuracy:50.44  POS: F1 Score:83.28 Accuracy:91.27 Loss:1.8504 , Time: 1412.8\n",
      "Steps: 8950 Loss:1.8474 , Current Loss: 1.8603\n",
      "Steps: 9050 Loss:1.8388 , Current Loss: 1.8396\n",
      "Steps: 9150 Loss:1.8321 , Current Loss: 1.7203\n",
      "Steps: 9250 Loss:1.8315 , Current Loss: 1.8536\n",
      "Steps: 9350 Loss:1.8275 , Current Loss: 1.8749\n",
      "Steps: 9450 Loss:1.8301 , Current Loss: 1.8099\n",
      "Steps: 9550 Loss:1.8240 , Current Loss: 1.7714\n",
      "Steps: 9650 Loss:1.8284 , Current Loss: 1.8471\n",
      "Epoch: 9 , Step: 9735 , loss: 1.8313 , Time: 6806.4\n",
      "Model Saved\n",
      "Steps: 9835 Loss:1.8154 , Current Loss: 1.8424\n",
      "Steps: 9935 Loss:1.8138 , Current Loss: 1.7877\n",
      "Steps: 10035 Loss:1.8125 , Current Loss: 1.7680\n",
      "Steps: 10135 Loss:1.8162 , Current Loss: 1.8843\n",
      "Steps: 10235 Loss:1.8140 , Current Loss: 1.7804\n",
      "Steps: 10335 Loss:1.8043 , Current Loss: 1.7837\n",
      "Steps: 10435 Loss:1.8078 , Current Loss: 1.8352\n",
      "Steps: 10535 Loss:1.8048 , Current Loss: 1.7421\n",
      "Epoch: 10 , Step: 10620 , loss: 1.8109 , Time: 6806.9\n",
      "Model Saved\n",
      "Val: F1 Score:42.33 Accuracy:51.38  POS: F1 Score:83.52 Accuracy:91.48 Loss:1.8129 , Time: 1413.8\n",
      "Steps: 10720 Loss:1.7900 , Current Loss: 1.7888\n",
      "Steps: 10820 Loss:1.7999 , Current Loss: 1.7591\n",
      "Steps: 10920 Loss:1.8008 , Current Loss: 1.7367\n",
      "Steps: 11020 Loss:1.7976 , Current Loss: 1.8162\n",
      "Steps: 11120 Loss:1.7934 , Current Loss: 1.8754\n",
      "Steps: 11220 Loss:1.7880 , Current Loss: 1.8686\n",
      "Steps: 11320 Loss:1.7931 , Current Loss: 1.8162\n",
      "Steps: 11420 Loss:1.7891 , Current Loss: 1.7141\n",
      "Epoch: 11 , Step: 11505 , loss: 1.7941 , Time: 6852.4\n",
      "Model Saved\n",
      "Steps: 11605 Loss:1.7776 , Current Loss: 1.8233\n",
      "Steps: 11705 Loss:1.7780 , Current Loss: 1.8208\n",
      "Steps: 11805 Loss:1.7908 , Current Loss: 1.8795\n",
      "Steps: 11905 Loss:1.7831 , Current Loss: 1.8047\n",
      "Steps: 12005 Loss:1.7732 , Current Loss: 1.7234\n",
      "Steps: 12105 Loss:1.7823 , Current Loss: 1.6746\n",
      "Steps: 12205 Loss:1.7745 , Current Loss: 1.8513\n",
      "Steps: 12305 Loss:1.7672 , Current Loss: 1.6707\n",
      "Epoch: 12 , Step: 12390 , loss: 1.7791 , Time: 6786.6\n",
      "Model Saved\n",
      "Val: F1 Score:43.67 Accuracy:52.23  POS: F1 Score:83.78 Accuracy:91.59 Loss:1.7894 , Time: 1408.2\n",
      "Steps: 12490 Loss:1.7689 , Current Loss: 1.7574\n",
      "Steps: 12590 Loss:1.7750 , Current Loss: 1.7044\n",
      "Steps: 12690 Loss:1.7715 , Current Loss: 1.8154\n",
      "Steps: 12790 Loss:1.7574 , Current Loss: 1.7210\n",
      "Steps: 12890 Loss:1.7636 , Current Loss: 1.8045\n",
      "Steps: 12990 Loss:1.7557 , Current Loss: 1.7883\n",
      "Steps: 13090 Loss:1.7556 , Current Loss: 1.7259\n",
      "Steps: 13190 Loss:1.7584 , Current Loss: 1.7887\n",
      "Epoch: 13 , Step: 13275 , loss: 1.7636 , Time: 6785.1\n",
      "Model Saved\n",
      "Steps: 13375 Loss:1.7607 , Current Loss: 1.6907\n",
      "Steps: 13475 Loss:1.7556 , Current Loss: 1.8272\n",
      "Steps: 13675 Loss:1.7565 , Current Loss: 1.6979\n",
      "Steps: 13775 Loss:1.7505 , Current Loss: 1.8146\n",
      "Steps: 13875 Loss:1.7434 , Current Loss: 1.7161\n",
      "Steps: 13975 Loss:1.7498 , Current Loss: 1.7632\n",
      "Steps: 14075 Loss:1.7403 , Current Loss: 1.7815\n",
      "Epoch: 14 , Step: 14160 , loss: 1.7513 , Time: 6800.4\n",
      "Model Saved\n",
      "Val: F1 Score:43.49 Accuracy:52.88  POS: F1 Score:83.84 Accuracy:91.67 Loss:1.7685 , Time: 1410.8\n",
      "Steps: 14260 Loss:1.7351 , Current Loss: 1.6896\n",
      "Steps: 14360 Loss:1.7401 , Current Loss: 1.7135\n",
      "Steps: 14460 Loss:1.7387 , Current Loss: 1.6994\n",
      "Steps: 14560 Loss:1.7393 , Current Loss: 1.7694\n",
      "Steps: 14660 Loss:1.7416 , Current Loss: 1.8355\n",
      "Steps: 14760 Loss:1.7343 , Current Loss: 1.7216\n",
      "Steps: 14860 Loss:1.7383 , Current Loss: 1.6986\n",
      "Steps: 14960 Loss:1.7350 , Current Loss: 1.6849\n",
      "Epoch: 15 , Step: 15045 , loss: 1.7381 , Time: 6797.3\n",
      "Model Saved\n",
      "Steps: 15145 Loss:1.7349 , Current Loss: 1.7787\n",
      "Steps: 15245 Loss:1.7292 , Current Loss: 1.7593\n",
      "Steps: 15345 Loss:1.7272 , Current Loss: 1.7042\n",
      "Steps: 15445 Loss:1.7283 , Current Loss: 1.7448\n",
      "Steps: 15545 Loss:1.7262 , Current Loss: 1.7450\n",
      "Steps: 15645 Loss:1.7268 , Current Loss: 1.7054\n",
      "Steps: 15745 Loss:1.7302 , Current Loss: 1.7167\n",
      "Steps: 15845 Loss:1.7272 , Current Loss: 1.7455\n",
      "Epoch: 16 , Step: 15930 , loss: 1.7280 , Time: 6797.8\n",
      "Model Saved\n",
      "Val: F1 Score:45.34 Accuracy:53.46  POS: F1 Score:83.96 Accuracy:91.78 Loss:1.7378 , Time: 1410.1\n",
      "Steps: 16030 Loss:1.7196 , Current Loss: 1.7225\n",
      "Steps: 16130 Loss:1.7222 , Current Loss: 1.7262\n",
      "Steps: 16230 Loss:1.7249 , Current Loss: 1.7665\n",
      "Steps: 16330 Loss:1.7185 , Current Loss: 1.7744\n",
      "Steps: 16430 Loss:1.7163 , Current Loss: 1.6369\n",
      "Steps: 16530 Loss:1.7174 , Current Loss: 1.7227\n",
      "Steps: 16630 Loss:1.7086 , Current Loss: 1.7435\n",
      "Steps: 16730 Loss:1.7181 , Current Loss: 1.6998\n",
      "Epoch: 17 , Step: 16815 , loss: 1.7181 , Time: 6780.6\n",
      "Model Saved\n",
      "Steps: 16915 Loss:1.7078 , Current Loss: 1.6998\n",
      "Steps: 17015 Loss:1.7103 , Current Loss: 1.7512\n",
      "Steps: 17115 Loss:1.7045 , Current Loss: 1.7422\n",
      "Steps: 17215 Loss:1.7183 , Current Loss: 1.8100\n",
      "Steps: 17315 Loss:1.7025 , Current Loss: 1.7655\n",
      "Steps: 17415 Loss:1.7080 , Current Loss: 1.7574\n",
      "Steps: 17515 Loss:1.7042 , Current Loss: 1.7543\n",
      "Steps: 17615 Loss:1.7029 , Current Loss: 1.7840\n",
      "Epoch: 18 , Step: 17700 , loss: 1.7070 , Time: 6788.0\n",
      "Model Saved\n",
      "Val: F1 Score:45.73 Accuracy:53.77  POS: F1 Score:84.25 Accuracy:91.88 Loss:1.7210 , Time: 1412.9\n",
      "Steps: 17800 Loss:1.6926 , Current Loss: 1.7028\n",
      "Steps: 17900 Loss:1.7041 , Current Loss: 1.7409\n",
      "Steps: 18000 Loss:1.6989 , Current Loss: 1.8232\n",
      "Steps: 18100 Loss:1.6923 , Current Loss: 1.6690\n",
      "Steps: 18200 Loss:1.7019 , Current Loss: 1.7806\n",
      "Steps: 18300 Loss:1.7001 , Current Loss: 1.6637\n",
      "Steps: 18400 Loss:1.7019 , Current Loss: 1.6625\n",
      "Steps: 18500 Loss:1.7025 , Current Loss: 1.5842\n",
      "Epoch: 19 , Step: 18585 , loss: 1.6991 , Time: 6805.1\n",
      "Model Saved\n",
      "Steps: 18685 Loss:1.6917 , Current Loss: 1.6917\n",
      "Steps: 18785 Loss:1.6922 , Current Loss: 1.6875\n",
      "Steps: 18885 Loss:1.6888 , Current Loss: 1.6877\n",
      "Steps: 18985 Loss:1.6836 , Current Loss: 1.6553\n",
      "Steps: 19085 Loss:1.6927 , Current Loss: 1.6342\n",
      "Steps: 19185 Loss:1.7014 , Current Loss: 1.7677\n",
      "Steps: 19285 Loss:1.6873 , Current Loss: 1.6640\n",
      "Steps: 19385 Loss:1.6910 , Current Loss: 1.7715\n",
      "Epoch: 20 , Step: 19470 , loss: 1.6903 , Time: 7074.6\n",
      "Model Saved\n",
      "Val: F1 Score:46.75 Accuracy:54.41  POS: F1 Score:84.32 Accuracy:91.95 Loss:1.7016 , Time: 1450.1\n"
     ]
    }
   ],
   "source": [
    "loss_collection = []\n",
    "val_collection = []\n",
    "num_epochs = 20\n",
    "val_period = 2\n",
    "\n",
    "# Pretraining POS Tags\n",
    "training(0, True)\n",
    "training(1, True)\n",
    "testing()\n",
    "\n",
    "for i in range(num_epochs):\n",
    "    loss_collection.append(training(i, False))\n",
    "    if((i+1)%val_period==0):\n",
    "        val_collection.append(testing())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Steps: 19570 Loss:1.6847 , Current Loss: 1.7190\n",
      "Steps: 19670 Loss:1.6807 , Current Loss: 1.6956\n",
      "Steps: 19770 Loss:1.6848 , Current Loss: 1.7595\n",
      "Steps: 19870 Loss:1.6813 , Current Loss: 1.6902\n",
      "Steps: 19970 Loss:1.6873 , Current Loss: 1.6548\n",
      "Steps: 20070 Loss:1.6830 , Current Loss: 1.6510\n",
      "Steps: 20170 Loss:1.6786 , Current Loss: 1.6772\n",
      "Steps: 20270 Loss:1.6851 , Current Loss: 1.7330\n",
      "Epoch: 1 , Step: 20355 , loss: 1.6824 , Time: 7379.7\n",
      "Model Saved\n",
      "Steps: 20455 Loss:1.6739 , Current Loss: 1.6805\n",
      "Steps: 20555 Loss:1.6755 , Current Loss: 1.5941\n",
      "Steps: 20655 Loss:1.6795 , Current Loss: 1.7452\n",
      "Steps: 20755 Loss:1.6735 , Current Loss: 1.6620\n",
      "Steps: 20855 Loss:1.6753 , Current Loss: 1.6128\n",
      "Steps: 20955 Loss:1.6675 , Current Loss: 1.7300\n",
      "Steps: 21055 Loss:1.6707 , Current Loss: 1.7664\n",
      "Steps: 21155 Loss:1.6739 , Current Loss: 1.6864\n",
      "Epoch: 2 , Step: 21240 , loss: 1.6733 , Time: 7256.1\n",
      "Model Saved\n",
      "Val: F1 Score:47.11 Accuracy:54.74  POS: F1 Score:84.34 Accuracy:92.01 Loss:1.6915 , Time: 1516.0\n",
      "Steps: 21340 Loss:1.6665 , Current Loss: 1.7193\n",
      "Steps: 21440 Loss:1.6713 , Current Loss: 1.5956\n"
     ]
    }
   ],
   "source": [
    "loss_collection = []\n",
    "val_collection = []\n",
    "num_epochs = 20\n",
    "val_period = 2\n",
    "\n",
    "for i in range(num_epochs):\n",
    "    loss_collection.append(training(i, False))\n",
    "    if((i+1)%val_period==0):\n",
    "        val_collection.append(testing())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Steps: 21540 Loss:1.6727 , Current Loss: 1.6282\n",
      "Steps: 21640 Loss:1.6661 , Current Loss: 1.6946\n",
      "Steps: 21740 Loss:1.6626 , Current Loss: 1.6765\n",
      "Steps: 21840 Loss:1.6655 , Current Loss: 1.6480\n",
      "Steps: 21940 Loss:1.6657 , Current Loss: 1.5697\n",
      "Steps: 22040 Loss:1.6689 , Current Loss: 1.6625\n",
      "Steps: 22140 Loss:1.6567 , Current Loss: 1.7403\n",
      "Steps: 22240 Loss:1.6624 , Current Loss: 1.6470\n",
      "Epoch: 1 , Step: 22325 , loss: 1.6654 , Time: 7334.5\n",
      "Model Saved\n",
      "Steps: 22425 Loss:1.6637 , Current Loss: 1.6799\n",
      "Steps: 22525 Loss:1.6653 , Current Loss: 1.7292\n",
      "Steps: 22625 Loss:1.6580 , Current Loss: 1.6184\n",
      "Steps: 22725 Loss:1.6658 , Current Loss: 1.7140\n",
      "Steps: 22825 Loss:1.6543 , Current Loss: 1.6572\n",
      "Steps: 22925 Loss:1.6609 , Current Loss: 1.6603\n",
      "Steps: 23025 Loss:1.6547 , Current Loss: 1.7083\n",
      "Steps: 23125 Loss:1.6467 , Current Loss: 1.6252\n",
      "Epoch: 2 , Step: 23210 , loss: 1.6587 , Time: 6984.0\n",
      "Model Saved\n",
      "Val: F1 Score:48.00 Accuracy:55.25  POS: F1 Score:84.50 Accuracy:92.09 Loss:1.6723 , Time: 1462.2\n",
      "Steps: 23310 Loss:1.6534 , Current Loss: 1.5971\n",
      "Steps: 23410 Loss:1.6667 , Current Loss: 1.7443\n",
      "Steps: 23510 Loss:1.6567 , Current Loss: 1.5834\n",
      "Steps: 23610 Loss:1.6579 , Current Loss: 1.7065\n",
      "Steps: 23710 Loss:1.6581 , Current Loss: 1.6879\n",
      "Steps: 23810 Loss:1.6431 , Current Loss: 1.7633\n",
      "Steps: 23910 Loss:1.6445 , Current Loss: 1.6376\n"
     ]
    }
   ],
   "source": [
    "loss_collection = []\n",
    "val_collection = []\n",
    "num_epochs = 20\n",
    "val_period = 2\n",
    "\n",
    "for i in range(num_epochs):\n",
    "    loss_collection.append(training(i, False))\n",
    "    if((i+1)%val_period==0):\n",
    "        val_collection.append(testing())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Steps: 24110 Loss:1.6476 , Current Loss: 1.6781\n",
      "Steps: 24210 Loss:1.6497 , Current Loss: 1.6395\n",
      "Steps: 24310 Loss:1.6547 , Current Loss: 1.6513\n",
      "Steps: 24410 Loss:1.6478 , Current Loss: 1.6324\n",
      "Steps: 24510 Loss:1.6439 , Current Loss: 1.7090\n",
      "Steps: 24610 Loss:1.6453 , Current Loss: 1.7414\n",
      "Steps: 24710 Loss:1.6399 , Current Loss: 1.6593\n",
      "Steps: 24810 Loss:1.6451 , Current Loss: 1.6259\n",
      "Epoch: 1 , Step: 24895 , loss: 1.6468 , Time: 7312.9\n",
      "Model Saved\n",
      "Steps: 24995 Loss:1.6371 , Current Loss: 1.6532\n",
      "Steps: 25095 Loss:1.6350 , Current Loss: 1.6565\n",
      "Steps: 25195 Loss:1.6392 , Current Loss: 1.5997\n",
      "Steps: 25295 Loss:1.6432 , Current Loss: 1.5932\n",
      "Steps: 25395 Loss:1.6387 , Current Loss: 1.6523\n",
      "Steps: 25495 Loss:1.6381 , Current Loss: 1.6335\n",
      "Steps: 25595 Loss:1.6416 , Current Loss: 1.6578\n",
      "Steps: 25695 Loss:1.6472 , Current Loss: 1.6469\n",
      "Epoch: 2 , Step: 25780 , loss: 1.6396 , Time: 7407.3\n",
      "Model Saved\n",
      "Val: F1 Score:48.04 Accuracy:55.58  POS: F1 Score:84.78 Accuracy:92.15 Loss:1.6527 , Time: 1472.4\n",
      "Steps: 25880 Loss:1.6420 , Current Loss: 1.6047\n",
      "Steps: 25980 Loss:1.6307 , Current Loss: 1.6127\n",
      "Steps: 26080 Loss:1.6337 , Current Loss: 1.6895\n",
      "Steps: 26180 Loss:1.6318 , Current Loss: 1.5832\n",
      "Steps: 26280 Loss:1.6418 , Current Loss: 1.5520\n",
      "Steps: 26380 Loss:1.6343 , Current Loss: 1.6225\n",
      "Steps: 26480 Loss:1.6333 , Current Loss: 1.6686\n",
      "Steps: 26580 Loss:1.6287 , Current Loss: 1.6301\n",
      "Epoch: 3 , Step: 26665 , loss: 1.6347 , Time: 7195.3\n",
      "Model Saved\n",
      "Steps: 26765 Loss:1.6432 , Current Loss: 1.6772\n",
      "Steps: 26865 Loss:1.6200 , Current Loss: 1.5762\n",
      "Steps: 26965 Loss:1.6196 , Current Loss: 1.6101\n",
      "Steps: 27065 Loss:1.6262 , Current Loss: 1.6512\n",
      "Steps: 27165 Loss:1.6271 , Current Loss: 1.5535\n",
      "Steps: 27265 Loss:1.6361 , Current Loss: 1.6622\n",
      "Steps: 27365 Loss:1.6388 , Current Loss: 1.6479\n",
      "Steps: 27465 Loss:1.6374 , Current Loss: 1.7183\n",
      "Epoch: 4 , Step: 27550 , loss: 1.6306 , Time: 7218.2\n",
      "Model Saved\n",
      "Val: F1 Score:48.52 Accuracy:55.94  POS: F1 Score:84.75 Accuracy:92.20 Loss:1.6532 , Time: 1600.8\n",
      "Steps: 27650 Loss:1.6111 , Current Loss: 1.5861\n",
      "Steps: 27750 Loss:1.6329 , Current Loss: 1.7225\n",
      "Steps: 27850 Loss:1.6276 , Current Loss: 1.6148\n",
      "Steps: 27950 Loss:1.6205 , Current Loss: 1.6590\n",
      "Steps: 28050 Loss:1.6174 , Current Loss: 1.6616\n",
      "Steps: 28150 Loss:1.6287 , Current Loss: 1.5895\n",
      "Steps: 28250 Loss:1.6314 , Current Loss: 1.6225\n",
      "Steps: 28350 Loss:1.6166 , Current Loss: 1.6575\n",
      "Epoch: 5 , Step: 28435 , loss: 1.6240 , Time: 7277.0\n",
      "Model Saved\n",
      "Steps: 28535 Loss:1.6136 , Current Loss: 1.6267\n",
      "Steps: 28635 Loss:1.6232 , Current Loss: 1.6718\n",
      "Steps: 28735 Loss:1.6165 , Current Loss: 1.6483\n",
      "Steps: 28835 Loss:1.6143 , Current Loss: 1.5861\n",
      "Steps: 28935 Loss:1.6203 , Current Loss: 1.6585\n",
      "Steps: 29035 Loss:1.6157 , Current Loss: 1.6076\n",
      "Steps: 29135 Loss:1.6213 , Current Loss: 1.6637\n",
      "Steps: 29235 Loss:1.6167 , Current Loss: 1.7090\n",
      "Epoch: 6 , Step: 29320 , loss: 1.6180 , Time: 8439.8\n",
      "Model Saved\n",
      "Val: F1 Score:48.79 Accuracy:56.02  POS: F1 Score:85.01 Accuracy:92.27 Loss:1.6332 , Time: 2099.8\n",
      "Steps: 29420 Loss:1.6123 , Current Loss: 1.5945\n",
      "Steps: 29520 Loss:1.6043 , Current Loss: 1.5816\n",
      "Steps: 29620 Loss:1.6117 , Current Loss: 1.5496\n",
      "Steps: 29720 Loss:1.6150 , Current Loss: 1.6055\n",
      "Steps: 29820 Loss:1.6159 , Current Loss: 1.6079\n",
      "Steps: 29920 Loss:1.6046 , Current Loss: 1.4933\n",
      "Steps: 30020 Loss:1.6142 , Current Loss: 1.5810\n",
      "Steps: 30120 Loss:1.6155 , Current Loss: 1.6673\n",
      "Epoch: 7 , Step: 30205 , loss: 1.6121 , Time: 9186.1\n",
      "Model Saved\n",
      "Steps: 30305 Loss:1.6121 , Current Loss: 1.5462\n",
      "Steps: 30405 Loss:1.6165 , Current Loss: 1.6605\n",
      "Steps: 30505 Loss:1.6110 , Current Loss: 1.5744\n",
      "Steps: 30605 Loss:1.6106 , Current Loss: 1.5631\n",
      "Steps: 30705 Loss:1.6102 , Current Loss: 1.5822\n",
      "Steps: 30805 Loss:1.6040 , Current Loss: 1.5579\n",
      "Steps: 30905 Loss:1.6068 , Current Loss: 1.5779\n",
      "Steps: 31005 Loss:1.6044 , Current Loss: 1.5519\n",
      "Epoch: 8 , Step: 31090 , loss: 1.6091 , Time: 8056.5\n",
      "Model Saved\n",
      "Val: F1 Score:49.25 Accuracy:56.38  POS: F1 Score:84.85 Accuracy:92.29 Loss:1.6276 , Time: 1409.2\n",
      "Steps: 31190 Loss:1.5998 , Current Loss: 1.6552\n",
      "Steps: 31290 Loss:1.6108 , Current Loss: 1.6569\n",
      "Steps: 31390 Loss:1.5998 , Current Loss: 1.6846\n",
      "Steps: 31490 Loss:1.5987 , Current Loss: 1.6832\n",
      "Steps: 31590 Loss:1.6094 , Current Loss: 1.5632\n",
      "Steps: 31690 Loss:1.6058 , Current Loss: 1.5658\n",
      "Steps: 31790 Loss:1.6088 , Current Loss: 1.5863\n",
      "Steps: 31890 Loss:1.6064 , Current Loss: 1.5993\n",
      "Epoch: 9 , Step: 31975 , loss: 1.6049 , Time: 7019.4\n",
      "Model Saved\n",
      "Steps: 32075 Loss:1.6083 , Current Loss: 1.6447\n",
      "Steps: 32175 Loss:1.5953 , Current Loss: 1.6170\n",
      "Steps: 32275 Loss:1.5982 , Current Loss: 1.6461\n",
      "Steps: 32375 Loss:1.5979 , Current Loss: 1.5243\n",
      "Steps: 32475 Loss:1.6016 , Current Loss: 1.6799\n",
      "Steps: 32575 Loss:1.5980 , Current Loss: 1.6598\n",
      "Steps: 32675 Loss:1.6088 , Current Loss: 1.6012\n",
      "Steps: 32775 Loss:1.6039 , Current Loss: 1.5445\n",
      "Epoch: 10 , Step: 32860 , loss: 1.6002 , Time: 7193.1\n",
      "Model Saved\n",
      "Val: F1 Score:49.74 Accuracy:56.58  POS: F1 Score:85.04 Accuracy:92.34 Loss:1.6254 , Time: 1577.9\n",
      "Steps: 32960 Loss:1.5906 , Current Loss: 1.5846\n",
      "Steps: 33060 Loss:1.5971 , Current Loss: 1.6788\n",
      "Steps: 33160 Loss:1.5885 , Current Loss: 1.6274\n",
      "Steps: 33260 Loss:1.5978 , Current Loss: 1.6183\n",
      "Steps: 33360 Loss:1.5958 , Current Loss: 1.6374\n",
      "Steps: 33460 Loss:1.5943 , Current Loss: 1.6958\n",
      "Steps: 33560 Loss:1.5925 , Current Loss: 1.5418\n",
      "Steps: 33660 Loss:1.6062 , Current Loss: 1.6335\n",
      "Epoch: 11 , Step: 33745 , loss: 1.5956 , Time: 7385.6\n",
      "Model Saved\n",
      "Steps: 33845 Loss:1.5924 , Current Loss: 1.6303\n",
      "Steps: 33945 Loss:1.5963 , Current Loss: 1.6427\n",
      "Steps: 34045 Loss:1.5983 , Current Loss: 1.6579\n",
      "Steps: 34145 Loss:1.5931 , Current Loss: 1.5771\n",
      "Steps: 34245 Loss:1.5968 , Current Loss: 1.5659\n",
      "Steps: 34345 Loss:1.5859 , Current Loss: 1.5213\n",
      "Steps: 34445 Loss:1.5907 , Current Loss: 1.6592\n",
      "Steps: 34545 Loss:1.5929 , Current Loss: 1.6248\n",
      "Epoch: 12 , Step: 34630 , loss: 1.5933 , Time: 7243.1\n",
      "Model Saved\n",
      "Val: F1 Score:49.66 Accuracy:56.76  POS: F1 Score:85.25 Accuracy:92.41 Loss:1.6118 , Time: 1514.1\n",
      "Steps: 34730 Loss:1.5862 , Current Loss: 1.6191\n",
      "Steps: 34830 Loss:1.5897 , Current Loss: 1.6075\n",
      "Steps: 34930 Loss:1.5896 , Current Loss: 1.5860\n",
      "Steps: 35030 Loss:1.5952 , Current Loss: 1.5000\n",
      "Steps: 35130 Loss:1.5864 , Current Loss: 1.5796\n",
      "Steps: 35230 Loss:1.5874 , Current Loss: 1.6735\n",
      "Steps: 35330 Loss:1.5946 , Current Loss: 1.6992\n",
      "Steps: 35430 Loss:1.5864 , Current Loss: 1.6102\n",
      "Epoch: 13 , Step: 35515 , loss: 1.5880 , Time: 7218.9\n",
      "Model Saved\n",
      "Steps: 35615 Loss:1.5894 , Current Loss: 1.4968\n",
      "Steps: 35715 Loss:1.5839 , Current Loss: 1.6190\n",
      "Steps: 35815 Loss:1.5811 , Current Loss: 1.6296\n",
      "Steps: 35915 Loss:1.5775 , Current Loss: 1.5823\n",
      "Steps: 36015 Loss:1.5861 , Current Loss: 1.5648\n",
      "Steps: 36115 Loss:1.5820 , Current Loss: 1.6754\n",
      "Steps: 36215 Loss:1.5762 , Current Loss: 1.5648\n",
      "Steps: 36315 Loss:1.5896 , Current Loss: 1.6276\n",
      "Epoch: 14 , Step: 36400 , loss: 1.5838 , Time: 7352.3\n",
      "Model Saved\n",
      "Val: F1 Score:49.81 Accuracy:56.84  POS: F1 Score:85.26 Accuracy:92.45 Loss:1.6092 , Time: 1557.8\n",
      "Steps: 36500 Loss:1.5715 , Current Loss: 1.5352\n",
      "Steps: 36600 Loss:1.5817 , Current Loss: 1.6570\n",
      "Steps: 36700 Loss:1.5843 , Current Loss: 1.5651\n",
      "Steps: 36800 Loss:1.5830 , Current Loss: 1.5537\n",
      "Steps: 36900 Loss:1.5799 , Current Loss: 1.6225\n",
      "Steps: 37000 Loss:1.5796 , Current Loss: 1.5629\n",
      "Steps: 37100 Loss:1.5798 , Current Loss: 1.4962\n",
      "Steps: 37200 Loss:1.5780 , Current Loss: 1.5804\n",
      "Epoch: 15 , Step: 37285 , loss: 1.5800 , Time: 7498.1\n",
      "Model Saved\n",
      "Steps: 37385 Loss:1.5837 , Current Loss: 1.5347\n",
      "Steps: 37485 Loss:1.5735 , Current Loss: 1.6400\n",
      "Steps: 37585 Loss:1.5752 , Current Loss: 1.5188\n",
      "Steps: 37685 Loss:1.5849 , Current Loss: 1.5767\n",
      "Steps: 37785 Loss:1.5762 , Current Loss: 1.6064\n",
      "Steps: 37885 Loss:1.5743 , Current Loss: 1.5673\n",
      "Steps: 37985 Loss:1.5754 , Current Loss: 1.4805\n",
      "Steps: 38085 Loss:1.5741 , Current Loss: 1.5066\n",
      "Epoch: 16 , Step: 38170 , loss: 1.5775 , Time: 7147.0\n",
      "Model Saved\n",
      "Val: F1 Score:49.96 Accuracy:57.07  POS: F1 Score:85.23 Accuracy:92.47 Loss:1.6132 , Time: 1520.1\n",
      "Steps: 38270 Loss:1.5691 , Current Loss: 1.6641\n",
      "Steps: 38370 Loss:1.5707 , Current Loss: 1.5942\n",
      "Steps: 38470 Loss:1.5758 , Current Loss: 1.6434\n",
      "Steps: 38570 Loss:1.5757 , Current Loss: 1.5976\n",
      "Steps: 38670 Loss:1.5724 , Current Loss: 1.6428\n",
      "Steps: 38770 Loss:1.5655 , Current Loss: 1.5517\n",
      "Steps: 38870 Loss:1.5751 , Current Loss: 1.5408\n",
      "Steps: 38970 Loss:1.5741 , Current Loss: 1.5982\n",
      "Epoch: 17 , Step: 39055 , loss: 1.5731 , Time: 8324.5\n",
      "Model Saved\n",
      "Steps: 39155 Loss:1.5714 , Current Loss: 1.5432\n",
      "Steps: 39255 Loss:1.5692 , Current Loss: 1.5282\n",
      "Steps: 39355 Loss:1.5682 , Current Loss: 1.5060\n",
      "Steps: 39455 Loss:1.5725 , Current Loss: 1.6055\n",
      "Steps: 39555 Loss:1.5674 , Current Loss: 1.5141\n",
      "Steps: 39655 Loss:1.5677 , Current Loss: 1.4970\n",
      "Steps: 39755 Loss:1.5650 , Current Loss: 1.5303\n",
      "Steps: 39855 Loss:1.5783 , Current Loss: 1.6041\n",
      "Epoch: 18 , Step: 39940 , loss: 1.5693 , Time: 7128.8\n",
      "Model Saved\n",
      "Val: F1 Score:50.31 Accuracy:57.14  POS: F1 Score:85.22 Accuracy:92.48 Loss:1.5930 , Time: 1499.3\n",
      "Steps: 40040 Loss:1.5686 , Current Loss: 1.5606\n",
      "Steps: 40140 Loss:1.5685 , Current Loss: 1.6783\n",
      "Steps: 40240 Loss:1.5726 , Current Loss: 1.5594\n",
      "Steps: 40340 Loss:1.5737 , Current Loss: 1.6103\n",
      "Steps: 40440 Loss:1.5657 , Current Loss: 1.4787\n",
      "Steps: 40540 Loss:1.5684 , Current Loss: 1.5504\n",
      "Steps: 40640 Loss:1.5669 , Current Loss: 1.5284\n",
      "Steps: 40740 Loss:1.5647 , Current Loss: 1.4905\n",
      "Epoch: 19 , Step: 40825 , loss: 1.5677 , Time: 7064.7\n",
      "Model Saved\n",
      "Steps: 40925 Loss:1.5790 , Current Loss: 1.5640\n",
      "Steps: 41025 Loss:1.5650 , Current Loss: 1.5415\n",
      "Steps: 41125 Loss:1.5662 , Current Loss: 1.5254\n",
      "Steps: 41225 Loss:1.5657 , Current Loss: 1.5834\n",
      "Steps: 41325 Loss:1.5664 , Current Loss: 1.5505\n",
      "Steps: 41425 Loss:1.5702 , Current Loss: 1.5970\n",
      "Steps: 41525 Loss:1.5658 , Current Loss: 1.5715\n",
      "Steps: 41625 Loss:1.5650 , Current Loss: 1.6048\n",
      "Epoch: 20 , Step: 41710 , loss: 1.5669 , Time: 7400.7\n",
      "Model Saved\n",
      "Val: F1 Score:50.80 Accuracy:57.47  POS: F1 Score:85.22 Accuracy:92.50 Loss:1.5767 , Time: 1581.5\n"
     ]
    }
   ],
   "source": [
    "loss_collection = []\n",
    "val_collection = []\n",
    "num_epochs = 20\n",
    "val_period = 2\n",
    "\n",
    "for i in range(num_epochs):\n",
    "    loss_collection.append(training(i, False))\n",
    "    if((i+1)%val_period==0):\n",
    "        val_collection.append(testing())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Steps: 42110 Loss:1.5642 , Current Loss: 1.5396\n",
      "Steps: 42210 Loss:1.5588 , Current Loss: 1.5635\n",
      "Steps: 42310 Loss:1.5649 , Current Loss: 1.6891\n",
      "Steps: 42410 Loss:1.5592 , Current Loss: 1.6037\n",
      "Steps: 42510 Loss:1.5621 , Current Loss: 1.5023\n",
      "Steps: 42610 Loss:1.5648 , Current Loss: 1.5459\n",
      "Steps: 42710 Loss:1.5668 , Current Loss: 1.5054\n",
      "Steps: 42810 Loss:1.5623 , Current Loss: 1.5792\n",
      "Epoch: 1 , Step: 42895 , loss: 1.5620 , Time: 7187.3\n",
      "Model Saved\n",
      "Steps: 42995 Loss:1.5526 , Current Loss: 1.5515\n",
      "Steps: 43095 Loss:1.5547 , Current Loss: 1.6026\n",
      "Steps: 43195 Loss:1.5632 , Current Loss: 1.5683\n",
      "Steps: 43295 Loss:1.5522 , Current Loss: 1.5982\n",
      "Steps: 43395 Loss:1.5634 , Current Loss: 1.5936\n",
      "Steps: 43495 Loss:1.5676 , Current Loss: 1.4807\n",
      "Steps: 43595 Loss:1.5654 , Current Loss: 1.6198\n",
      "Steps: 43695 Loss:1.5627 , Current Loss: 1.5147\n",
      "Epoch: 2 , Step: 43780 , loss: 1.5598 , Time: 6778.8\n",
      "Model Saved\n",
      "Val: F1 Score:50.86 Accuracy:57.77  POS: F1 Score:85.34 Accuracy:92.52 Loss:1.5774 , Time: 1408.8\n",
      "Steps: 43880 Loss:1.5572 , Current Loss: 1.6304\n",
      "Steps: 43980 Loss:1.5583 , Current Loss: 1.5412\n",
      "Steps: 44080 Loss:1.5519 , Current Loss: 1.5677\n",
      "Steps: 44180 Loss:1.5574 , Current Loss: 1.6106\n",
      "Steps: 44280 Loss:1.5536 , Current Loss: 1.4888\n",
      "Steps: 44380 Loss:1.5468 , Current Loss: 1.4949\n",
      "Steps: 44480 Loss:1.5589 , Current Loss: 1.5207\n",
      "Steps: 44580 Loss:1.5481 , Current Loss: 1.5092\n",
      "Epoch: 3 , Step: 44665 , loss: 1.5536 , Time: 6751.4\n",
      "Model Saved\n",
      "Steps: 44765 Loss:1.5499 , Current Loss: 1.5121\n",
      "Steps: 44865 Loss:1.5561 , Current Loss: 1.5341\n",
      "Steps: 44965 Loss:1.5429 , Current Loss: 1.5500\n",
      "Steps: 45065 Loss:1.5519 , Current Loss: 1.5013\n",
      "Steps: 45165 Loss:1.5492 , Current Loss: 1.5110\n",
      "Steps: 45265 Loss:1.5424 , Current Loss: 1.5359\n",
      "Steps: 45365 Loss:1.5538 , Current Loss: 1.5465\n",
      "Steps: 45465 Loss:1.5615 , Current Loss: 1.5559\n",
      "Epoch: 4 , Step: 45550 , loss: 1.5514 , Time: 6798.7\n",
      "Model Saved\n",
      "Val: F1 Score:50.93 Accuracy:57.76  POS: F1 Score:85.26 Accuracy:92.57 Loss:1.5729 , Time: 1415.6\n",
      "Steps: 45650 Loss:1.5566 , Current Loss: 1.5575\n",
      "Steps: 45750 Loss:1.5507 , Current Loss: 1.4424\n",
      "Steps: 45850 Loss:1.5528 , Current Loss: 1.4849\n",
      "Steps: 45950 Loss:1.5570 , Current Loss: 1.5017\n",
      "Steps: 46050 Loss:1.5488 , Current Loss: 1.5334\n",
      "Steps: 46150 Loss:1.5520 , Current Loss: 1.4573\n",
      "Steps: 46250 Loss:1.5486 , Current Loss: 1.4874\n",
      "Steps: 46350 Loss:1.5480 , Current Loss: 1.6767\n",
      "Epoch: 5 , Step: 46435 , loss: 1.5513 , Time: 6790.8\n",
      "Model Saved\n",
      "Steps: 46535 Loss:1.5453 , Current Loss: 1.5583\n",
      "Steps: 46635 Loss:1.5435 , Current Loss: 1.5153\n",
      "Steps: 46735 Loss:1.5489 , Current Loss: 1.5856\n",
      "Steps: 46835 Loss:1.5465 , Current Loss: 1.4792\n",
      "Steps: 46935 Loss:1.5452 , Current Loss: 1.5074\n",
      "Steps: 47035 Loss:1.5467 , Current Loss: 1.5457\n",
      "Steps: 47135 Loss:1.5424 , Current Loss: 1.6263\n",
      "Steps: 47235 Loss:1.5485 , Current Loss: 1.5101\n",
      "Epoch: 6 , Step: 47320 , loss: 1.5461 , Time: 6803.1\n",
      "Model Saved\n",
      "Val: F1 Score:51.02 Accuracy:57.81  POS: F1 Score:85.34 Accuracy:92.60 Loss:1.5627 , Time: 1421.6\n",
      "Steps: 47420 Loss:1.5427 , Current Loss: 1.5293\n",
      "Steps: 47520 Loss:1.5357 , Current Loss: 1.6023\n",
      "Steps: 47620 Loss:1.5505 , Current Loss: 1.4999\n",
      "Steps: 47720 Loss:1.5449 , Current Loss: 1.5035\n",
      "Steps: 47820 Loss:1.5449 , Current Loss: 1.5059\n",
      "Steps: 47920 Loss:1.5434 , Current Loss: 1.5918\n",
      "Steps: 48020 Loss:1.5391 , Current Loss: 1.4501\n",
      "Steps: 48120 Loss:1.5422 , Current Loss: 1.5972\n",
      "Epoch: 7 , Step: 48205 , loss: 1.5429 , Time: 6807.8\n",
      "Model Saved\n",
      "Steps: 48305 Loss:1.5468 , Current Loss: 1.5573\n",
      "Steps: 48405 Loss:1.5398 , Current Loss: 1.5863\n",
      "Steps: 48505 Loss:1.5494 , Current Loss: 1.6224\n",
      "Steps: 48605 Loss:1.5391 , Current Loss: 1.5857\n",
      "Steps: 48705 Loss:1.5411 , Current Loss: 1.5570\n",
      "Steps: 48805 Loss:1.5433 , Current Loss: 1.5445\n",
      "Steps: 48905 Loss:1.5374 , Current Loss: 1.5132\n",
      "Steps: 49005 Loss:1.5401 , Current Loss: 1.4743\n",
      "Epoch: 8 , Step: 49090 , loss: 1.5411 , Time: 6814.7\n",
      "Model Saved\n",
      "Val: F1 Score:51.40 Accuracy:57.86  POS: F1 Score:85.44 Accuracy:92.63 Loss:1.5663 , Time: 1421.1\n",
      "Steps: 49190 Loss:1.5371 , Current Loss: 1.6540\n",
      "Steps: 49290 Loss:1.5455 , Current Loss: 1.5865\n",
      "Steps: 49390 Loss:1.5384 , Current Loss: 1.5762\n",
      "Steps: 49490 Loss:1.5379 , Current Loss: 1.4426\n",
      "Steps: 49590 Loss:1.5436 , Current Loss: 1.5179\n",
      "Steps: 49690 Loss:1.5339 , Current Loss: 1.5149\n",
      "Steps: 49790 Loss:1.5422 , Current Loss: 1.5335\n",
      "Steps: 49890 Loss:1.5273 , Current Loss: 1.5221\n",
      "Epoch: 9 , Step: 49975 , loss: 1.5381 , Time: 6813.9\n",
      "Model Saved\n",
      "Steps: 50075 Loss:1.5302 , Current Loss: 1.5917\n",
      "Steps: 50175 Loss:1.5284 , Current Loss: 1.4397\n",
      "Steps: 50275 Loss:1.5383 , Current Loss: 1.5201\n",
      "Steps: 50375 Loss:1.5427 , Current Loss: 1.4793\n",
      "Steps: 50475 Loss:1.5314 , Current Loss: 1.5344\n",
      "Steps: 50575 Loss:1.5388 , Current Loss: 1.5248\n",
      "Steps: 50675 Loss:1.5341 , Current Loss: 1.5310\n",
      "Steps: 50775 Loss:1.5379 , Current Loss: 1.5672\n",
      "Epoch: 10 , Step: 50860 , loss: 1.5357 , Time: 6830.2\n",
      "Model Saved\n",
      "Val: F1 Score:51.60 Accuracy:58.18  POS: F1 Score:85.51 Accuracy:92.63 Loss:1.5628 , Time: 1421.5\n",
      "Steps: 50960 Loss:1.5411 , Current Loss: 1.5196\n",
      "Steps: 51060 Loss:1.5364 , Current Loss: 1.5618\n",
      "Steps: 51160 Loss:1.5313 , Current Loss: 1.5587\n",
      "Steps: 51260 Loss:1.5313 , Current Loss: 1.4965\n",
      "Steps: 51360 Loss:1.5242 , Current Loss: 1.5015\n",
      "Steps: 51460 Loss:1.5319 , Current Loss: 1.4889\n",
      "Steps: 51560 Loss:1.5256 , Current Loss: 1.5206\n",
      "Steps: 51660 Loss:1.5325 , Current Loss: 1.5722\n",
      "Epoch: 11 , Step: 51745 , loss: 1.5313 , Time: 6810.5\n",
      "Model Saved\n",
      "Steps: 51845 Loss:1.5388 , Current Loss: 1.4573\n",
      "Steps: 51945 Loss:1.5287 , Current Loss: 1.5344\n",
      "Steps: 52045 Loss:1.5288 , Current Loss: 1.4396\n",
      "Steps: 52145 Loss:1.5318 , Current Loss: 1.6263\n",
      "Steps: 52245 Loss:1.5390 , Current Loss: 1.5872\n",
      "Steps: 52345 Loss:1.5267 , Current Loss: 1.4402\n",
      "Steps: 52445 Loss:1.5397 , Current Loss: 1.4800\n",
      "Steps: 52545 Loss:1.5261 , Current Loss: 1.4781\n",
      "Epoch: 12 , Step: 52630 , loss: 1.5321 , Time: 6804.8\n",
      "Model Saved\n",
      "Val: F1 Score:51.87 Accuracy:58.27  POS: F1 Score:85.37 Accuracy:92.65 Loss:1.5606 , Time: 1421.1\n",
      "Steps: 52730 Loss:1.5359 , Current Loss: 1.5225\n",
      "Steps: 52830 Loss:1.5300 , Current Loss: 1.5302\n",
      "Steps: 52930 Loss:1.5322 , Current Loss: 1.5791\n",
      "Steps: 53030 Loss:1.5254 , Current Loss: 1.5649\n",
      "Steps: 53130 Loss:1.5384 , Current Loss: 1.5680\n",
      "Steps: 53230 Loss:1.5320 , Current Loss: 1.5011\n",
      "Steps: 53330 Loss:1.5310 , Current Loss: 1.4922\n",
      "Steps: 53430 Loss:1.5291 , Current Loss: 1.5706\n",
      "Epoch: 13 , Step: 53515 , loss: 1.5305 , Time: 6807.1\n",
      "Model Saved\n",
      "Steps: 53615 Loss:1.5277 , Current Loss: 1.6009\n",
      "Steps: 53715 Loss:1.5288 , Current Loss: 1.5429\n",
      "Steps: 53815 Loss:1.5259 , Current Loss: 1.5426\n",
      "Steps: 53915 Loss:1.5360 , Current Loss: 1.5399\n",
      "Steps: 54015 Loss:1.5126 , Current Loss: 1.5003\n",
      "Steps: 54115 Loss:1.5269 , Current Loss: 1.5018\n",
      "Steps: 54215 Loss:1.5157 , Current Loss: 1.4318\n",
      "Steps: 54315 Loss:1.5301 , Current Loss: 1.5310\n",
      "Epoch: 14 , Step: 54400 , loss: 1.5247 , Time: 6780.2\n",
      "Model Saved\n",
      "Val: F1 Score:52.02 Accuracy:58.17  POS: F1 Score:85.51 Accuracy:92.67 Loss:1.5525 , Time: 1419.5\n",
      "Steps: 54500 Loss:1.5243 , Current Loss: 1.5732\n",
      "Steps: 54600 Loss:1.5262 , Current Loss: 1.6062\n",
      "Steps: 54700 Loss:1.5316 , Current Loss: 1.5436\n",
      "Steps: 54800 Loss:1.5168 , Current Loss: 1.5092\n",
      "Steps: 54900 Loss:1.5212 , Current Loss: 1.5415\n",
      "Steps: 55000 Loss:1.5301 , Current Loss: 1.5390\n",
      "Steps: 55100 Loss:1.5257 , Current Loss: 1.4548\n",
      "Steps: 55200 Loss:1.5287 , Current Loss: 1.4379\n",
      "Epoch: 15 , Step: 55285 , loss: 1.5253 , Time: 6798.0\n",
      "Model Saved\n",
      "Steps: 55385 Loss:1.5286 , Current Loss: 1.5398\n",
      "Steps: 55485 Loss:1.5169 , Current Loss: 1.5834\n",
      "Steps: 55585 Loss:1.5169 , Current Loss: 1.5762\n",
      "Steps: 55685 Loss:1.5291 , Current Loss: 1.6792\n",
      "Steps: 55785 Loss:1.5219 , Current Loss: 1.4798\n",
      "Steps: 55885 Loss:1.5250 , Current Loss: 1.5352\n",
      "Steps: 55985 Loss:1.5331 , Current Loss: 1.5155\n",
      "Steps: 56085 Loss:1.5250 , Current Loss: 1.5038\n",
      "Epoch: 16 , Step: 56170 , loss: 1.5243 , Time: 6794.7\n",
      "Model Saved\n",
      "Val: F1 Score:52.16 Accuracy:58.49  POS: F1 Score:85.59 Accuracy:92.70 Loss:1.5424 , Time: 1413.4\n",
      "Steps: 56270 Loss:1.5173 , Current Loss: 1.4373\n",
      "Steps: 56370 Loss:1.5222 , Current Loss: 1.4808\n",
      "Steps: 56470 Loss:1.5248 , Current Loss: 1.5182\n",
      "Steps: 56570 Loss:1.5090 , Current Loss: 1.5499\n",
      "Steps: 56670 Loss:1.5215 , Current Loss: 1.5554\n",
      "Steps: 56770 Loss:1.5207 , Current Loss: 1.4954\n",
      "Steps: 56870 Loss:1.5212 , Current Loss: 1.5122\n",
      "Steps: 56970 Loss:1.5156 , Current Loss: 1.5429\n",
      "Epoch: 17 , Step: 57055 , loss: 1.5189 , Time: 6807.6\n",
      "Model Saved\n",
      "Steps: 57155 Loss:1.5244 , Current Loss: 1.6389\n",
      "Steps: 57255 Loss:1.5164 , Current Loss: 1.5099\n",
      "Steps: 57355 Loss:1.5133 , Current Loss: 1.5458\n",
      "Steps: 57455 Loss:1.5220 , Current Loss: 1.4820\n",
      "Steps: 57555 Loss:1.5227 , Current Loss: 1.5529\n",
      "Steps: 57655 Loss:1.5076 , Current Loss: 1.5213\n",
      "Steps: 57755 Loss:1.5234 , Current Loss: 1.5431\n",
      "Steps: 57855 Loss:1.5242 , Current Loss: 1.4532\n",
      "Epoch: 18 , Step: 57940 , loss: 1.5197 , Time: 6803.5\n",
      "Model Saved\n",
      "Val: F1 Score:52.04 Accuracy:58.41  POS: F1 Score:85.54 Accuracy:92.71 Loss:1.5516 , Time: 1413.3\n",
      "Steps: 58040 Loss:1.5128 , Current Loss: 1.4500\n",
      "Steps: 58140 Loss:1.5122 , Current Loss: 1.5573\n",
      "Steps: 58240 Loss:1.5129 , Current Loss: 1.4911\n",
      "Steps: 58340 Loss:1.5161 , Current Loss: 1.5652\n",
      "Steps: 58440 Loss:1.5123 , Current Loss: 1.5627\n",
      "Steps: 58540 Loss:1.5114 , Current Loss: 1.5808\n",
      "Steps: 58640 Loss:1.5197 , Current Loss: 1.5270\n",
      "Steps: 58740 Loss:1.5311 , Current Loss: 1.6027\n",
      "Epoch: 19 , Step: 58825 , loss: 1.5156 , Time: 6808.6\n",
      "Model Saved\n",
      "Steps: 58925 Loss:1.5178 , Current Loss: 1.5591\n",
      "Steps: 59025 Loss:1.5123 , Current Loss: 1.5507\n",
      "Steps: 59125 Loss:1.5201 , Current Loss: 1.5506\n",
      "Steps: 59225 Loss:1.5172 , Current Loss: 1.5787\n",
      "Steps: 59325 Loss:1.5198 , Current Loss: 1.4983\n",
      "Steps: 59425 Loss:1.5177 , Current Loss: 1.5450\n",
      "Steps: 59525 Loss:1.5061 , Current Loss: 1.4430\n",
      "Steps: 59625 Loss:1.5088 , Current Loss: 1.5192\n",
      "Epoch: 20 , Step: 59710 , loss: 1.5160 , Time: 6806.5\n",
      "Model Saved\n",
      "Val: F1 Score:52.19 Accuracy:58.68  POS: F1 Score:85.66 Accuracy:92.72 Loss:1.5449 , Time: 1418.2\n"
     ]
    }
   ],
   "source": [
    "loss_collection = []\n",
    "val_collection = []\n",
    "num_epochs = 20\n",
    "val_period = 2\n",
    "\n",
    "for i in range(num_epochs):\n",
    "    loss_collection.append(training(i, False))\n",
    "    if((i+1)%val_period==0):\n",
    "        val_collection.append(testing())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "start_time = time.time()\n",
    "train_loss, train_pred, train_true, train_pred_pos, train_true_pos = model(x_id_train, y_train, y_pos_train, mask_train, sense_mask_train, train_cond=False)        \n",
    "f1_, accu_, f1_pos_, accu_pos_ = etrain_score(train_true, train_pred, train_true_pos, train_pred_pos)\n",
    "time_taken = time.time() - start_time\n",
    "print(\"train: F1 Score:{0:.2f}\".format(f1_), \"Accuracy:{0:.2f}\".format(accu_), \" POS: F1 Score:{0:.2f}\".format(f1_pos_), \"Accuracy:{0:.2f}\".format(accu_pos_), \"Loss:{0:.4f}\".format(train_loss), \", Time: {0:.1f}\".format(time_taken))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "saver.restore(sess, save_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "env",
   "language": "python",
   "name": "env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}