one_million/all-word/Model-aw-lex-local_attention-fast-v3-1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### fast local attention + crf \n",
    "\n",
    "#### batch size 128, learning rate = 0.001, windows size 5, num_gpus = 6\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "tf.logging.set_verbosity(tf.logging.WARN)\n",
    "import pickle\n",
    "import numpy as np\n",
    "import os\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import accuracy_score\n",
    "import os\n",
    "from tensorflow.python.client import device_lib\n",
    "from collections import Counter\n",
    "import time\n",
    "\n",
    "VERY_BIG_NUMBER = 1e30"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = open('../../Glove/word_embedding_glove', 'rb')\n",
    "word_embedding = pickle.load(f)\n",
    "f.close()\n",
    "\n",
    "word_embedding = word_embedding[: len(word_embedding)-1]\n",
    "\n",
    "f = open('../../Glove/vocab_glove', 'rb')\n",
    "vocab = pickle.load(f)\n",
    "f.close()\n",
    "\n",
    "word2id = dict((w, i) for i,w in enumerate(vocab))\n",
    "id2word = dict((i, w) for i,w in enumerate(vocab))\n",
    "\n",
    "unknown_token = \"UNKNOWN_TOKEN\"\n",
    "\n",
    "# Model Description\n",
    "model_name = 'model-aw-lex-local-att-fast-v3-1'\n",
    "model_dir = '../output/all-word/' + model_name\n",
    "save_dir = os.path.join(model_dir, \"save/\")\n",
    "log_dir = os.path.join(model_dir, \"log\")\n",
    "\n",
    "if not os.path.exists(model_dir):\n",
    "    os.mkdir(model_dir)\n",
    "if not os.path.exists(save_dir):\n",
    "    os.mkdir(save_dir)\n",
    "if not os.path.exists(log_dir):\n",
    "    os.mkdir(log_dir)\n",
    "\n",
    "with open('../../../dataset/train_val_data_fine/all_word_lex','rb') as f:\n",
    "    train_data, val_data = pickle.load(f)    \n",
    "    \n",
    "# Parameters\n",
    "mode = 'train'\n",
    "num_senses = 45\n",
    "num_pos = 12\n",
    "batch_size = 128\n",
    "vocab_size = len(vocab)\n",
    "unk_vocab_size = 1\n",
    "word_emb_size = len(word_embedding[0])\n",
    "max_sent_size = 200\n",
    "hidden_size = 256\n",
    "num_filter = 256\n",
    "window_size = 5\n",
    "kernel_size = 5\n",
    "keep_prob = 0.3\n",
    "l2_lambda = 0.001\n",
    "init_lr = 0.001\n",
    "decay_steps = 500\n",
    "decay_rate = 0.99\n",
    "clip_norm = 1\n",
    "clipping = True\n",
    "crf_lambda = 0.05\n",
    "moving_avg_deacy = 0.999\n",
    "num_gpus = 6\n",
    "width = int(window_size/2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def average_gradients(tower_grads):\n",
    "    average_grads = []\n",
    "    for grad_and_vars in zip(*tower_grads):\n",
    "        # Note that each grad_and_vars looks like the following:\n",
    "        #   ((grad0_gpu0, var0_gpu0), ... , (grad0_gpuN, var0_gpuN))\n",
    "        grads = []\n",
    "        for g, _ in grad_and_vars:\n",
    "            # Add 0 dimension to the gradients to represent the tower.\n",
    "            expanded_g = tf.expand_dims(g, 0)\n",
    "\n",
    "            # Append on a 'tower' dimension which we will average over below.\n",
    "            grads.append(expanded_g)\n",
    "\n",
    "        # Average over the 'tower' dimension.\n",
    "        grad = tf.concat(grads, 0)\n",
    "        grad = tf.reduce_mean(grad, axis=0)\n",
    "\n",
    "        # Keep in mind that the Variables are redundant because they are shared\n",
    "        # across towers. So .. we will just return the first tower's pointer to\n",
    "        # the Variable.\n",
    "        v = grad_and_vars[0][1]\n",
    "        grad_and_var = (grad, v)\n",
    "        average_grads.append(grad_and_var)\n",
    "    return average_grads"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/clip_ops.py:110: calling reduce_sum (from tensorflow.python.ops.math_ops) with keep_dims is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "keep_dims is deprecated, use keepdims instead\n"
     ]
    }
   ],
   "source": [
    "# MODEL\n",
    "device_num = 0\n",
    "tower_grads = []\n",
    "losses = []\n",
    "predictions = []\n",
    "predictions_pos = []\n",
    "total_trans_params = []\n",
    "\n",
    "x = tf.placeholder('int32', [num_gpus, batch_size, max_sent_size], name=\"x\")\n",
    "y = tf.placeholder('int32', [num_gpus, batch_size, max_sent_size], name=\"y\")\n",
    "y_pos = tf.placeholder('int32', [num_gpus, batch_size, max_sent_size], name=\"y\")\n",
    "x_mask  = tf.placeholder('bool', [num_gpus, batch_size, max_sent_size], name='x_mask') \n",
    "sense_mask  = tf.placeholder('bool', [num_gpus, batch_size, max_sent_size], name='sense_mask')\n",
    "is_train = tf.placeholder('bool', [], name='is_train')\n",
    "word_emb_mat = tf.placeholder('float', [None, word_emb_size], name='emb_mat')\n",
    "input_keep_prob = tf.cond(is_train,lambda:keep_prob, lambda:tf.constant(1.0))\n",
    "pretrain = tf.placeholder('bool', [], name=\"pretrain\")\n",
    "\n",
    "global_step = tf.Variable(0, trainable=False, name=\"global_step\")\n",
    "learning_rate = tf.train.exponential_decay(init_lr, global_step, decay_steps, decay_rate, staircase=True)\n",
    "summaries = []\n",
    "\n",
    "def global_attention(input_x, input_mask, W_att):\n",
    "    h_masked = tf.boolean_mask(input_x, input_mask)\n",
    "    h_tanh = tf.tanh(h_masked)\n",
    "    u = tf.matmul(h_tanh, W_att)\n",
    "    a = tf.nn.softmax(u)\n",
    "    c = tf.reduce_sum(tf.multiply(h_masked, a), axis=0)  \n",
    "    return c\n",
    "\n",
    "with tf.variable_scope(\"word_embedding\"):\n",
    "    unk_word_emb_mat = tf.get_variable(\"word_emb_mat\", dtype='float', shape=[unk_vocab_size, word_emb_size], initializer=tf.contrib.layers.xavier_initializer(uniform=True, seed=0, dtype=tf.float32))\n",
    "    final_word_emb_mat = tf.concat([word_emb_mat, unk_word_emb_mat], 0)\n",
    "\n",
    "with tf.variable_scope(tf.get_variable_scope()):\n",
    "    for gpu_idx in range(num_gpus):\n",
    "        if gpu_idx>=3:\n",
    "            device_num = 1\n",
    "        with tf.name_scope(\"model_{}\".format(gpu_idx)) as scope, tf.device('/gpu:%d' % device_num):\n",
    "\n",
    "            if gpu_idx > 0:\n",
    "                    tf.get_variable_scope().reuse_variables()\n",
    "\n",
    "            with tf.name_scope(\"word\"):\n",
    "                Wx = tf.nn.embedding_lookup(final_word_emb_mat, x[gpu_idx])  \n",
    "\n",
    "            float_x_mask = tf.cast(x_mask[gpu_idx], 'float')\n",
    "            x_len = tf.reduce_sum(tf.cast(x_mask[gpu_idx], 'int32'), axis=1)\n",
    "            \n",
    "            with tf.variable_scope(\"lstm1\"):\n",
    "                cell_fw1 = tf.contrib.rnn.BasicLSTMCell(hidden_size,state_is_tuple=True)\n",
    "                cell_bw1 = tf.contrib.rnn.BasicLSTMCell(hidden_size,state_is_tuple=True)\n",
    "\n",
    "                d_cell_fw1 = tf.contrib.rnn.DropoutWrapper(cell_fw1, input_keep_prob=input_keep_prob)\n",
    "                d_cell_bw1 = tf.contrib.rnn.DropoutWrapper(cell_bw1, input_keep_prob=input_keep_prob)\n",
    "\n",
    "                (fw_h1, bw_h1), _ = tf.nn.bidirectional_dynamic_rnn(d_cell_fw1, d_cell_bw1, Wx, sequence_length=x_len, dtype='float', scope='lstm1')\n",
    "                h1 = tf.concat([fw_h1, bw_h1], 2)\n",
    "\n",
    "            with tf.variable_scope(\"lstm2\"):\n",
    "                cell_fw2 = tf.contrib.rnn.BasicLSTMCell(hidden_size,state_is_tuple=True)\n",
    "                cell_bw2 = tf.contrib.rnn.BasicLSTMCell(hidden_size,state_is_tuple=True)\n",
    "\n",
    "                d_cell_fw2 = tf.contrib.rnn.DropoutWrapper(cell_fw2, input_keep_prob=input_keep_prob)\n",
    "                d_cell_bw2 = tf.contrib.rnn.DropoutWrapper(cell_bw2, input_keep_prob=input_keep_prob)\n",
    "\n",
    "                (fw_h2, bw_h2), _ = tf.nn.bidirectional_dynamic_rnn(d_cell_fw2, d_cell_bw2, h1, sequence_length=x_len, dtype='float', scope='lstm2')\n",
    "                h = tf.concat([fw_h2, bw_h2], 2)\n",
    "            \n",
    "            with tf.variable_scope(\"local_attention\"):\n",
    "                W_att_local = tf.get_variable(\"W_att_local\", shape=[2*hidden_size, 1], initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1, seed=gpu_idx*10))\n",
    "                flat_h = tf.reshape(h, [batch_size*max_sent_size, tf.shape(h)[2]])\n",
    "                h_tanh = tf.tanh(flat_h)\n",
    "                u_flat = tf.matmul(h_tanh, W_att_local)\n",
    "                u_local = tf.reshape(u_flat, [batch_size, max_sent_size])\n",
    "                final_u = (tf.cast(x_mask[gpu_idx], 'float') -1)*VERY_BIG_NUMBER + u_local\n",
    "                c_local = tf.map_fn(lambda i:tf.reduce_sum(tf.multiply(h[:, tf.maximum(0, i-width-1):tf.minimum(1+width+i, max_sent_size)],\n",
    "                    tf.expand_dims(tf.nn.softmax(final_u[:, tf.maximum(0, i-width-1):tf.minimum(1+width+i, max_sent_size)], 1), 2)), axis=1),\n",
    "                    tf.range(max_sent_size), dtype=tf.float32)  \n",
    "                \n",
    "            c_local = tf.transpose(c_local, perm=[1,0,2]) \n",
    "            h_final = tf.concat([tf.multiply(c_local, tf.expand_dims(float_x_mask, 2)), h], 2)\n",
    "            flat_h_final = tf.reshape(h_final, [-1, tf.shape(h_final)[2]])\n",
    "    \n",
    "            \n",
    "            with tf.variable_scope(\"hidden_layer\"):\n",
    "                W = tf.get_variable(\"W\", shape=[4*hidden_size, 2*hidden_size], initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1, seed=gpu_idx*20))\n",
    "                b = tf.get_variable(\"b\", shape=[2*hidden_size], initializer=tf.zeros_initializer())\n",
    "                drop_flat_h_final = tf.nn.dropout(flat_h_final, input_keep_prob)\n",
    "                flat_hl = tf.matmul(drop_flat_h_final, W) + b\n",
    "\n",
    "            with tf.variable_scope(\"softmax_layer\"):\n",
    "                W = tf.get_variable(\"W\", shape=[2*hidden_size, num_senses], initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1, seed=gpu_idx*20))\n",
    "                b = tf.get_variable(\"b\", shape=[num_senses], initializer=tf.zeros_initializer())\n",
    "                drop_flat_hl = tf.nn.dropout(flat_hl, input_keep_prob)\n",
    "                flat_logits_sense = tf.matmul(drop_flat_hl, W) + b\n",
    "                logits = tf.reshape(flat_logits_sense, [batch_size, max_sent_size, num_senses])\n",
    "                predictions.append(tf.argmax(logits, 2))\n",
    "\n",
    "            with tf.variable_scope(\"softmax_layer_pos\"):\n",
    "                W = tf.get_variable(\"W\", shape=[2*hidden_size, num_pos], initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1, seed=gpu_idx*30))\n",
    "                b = tf.get_variable(\"b\", shape=[num_pos], initializer=tf.zeros_initializer())\n",
    "                flat_h1 = tf.reshape(h1, [-1, tf.shape(h1)[2]])\n",
    "                drop_flat_hl = tf.nn.dropout(flat_hl, input_keep_prob)\n",
    "                flat_logits_pos = tf.matmul(drop_flat_hl, W) + b\n",
    "                logits_pos = tf.reshape(flat_logits_pos, [batch_size, max_sent_size, num_pos])\n",
    "                log_likelihood, trans_params = tf.contrib.crf.crf_log_likelihood(logits_pos, y_pos[gpu_idx], x_len)\n",
    "                loss_pos = crf_lambda*tf.reduce_mean(-log_likelihood)\n",
    "                predictions_pos.append(logits_pos)\n",
    "                total_trans_params.append(trans_params)\n",
    "\n",
    "            float_sense_mask = tf.cast(sense_mask[gpu_idx], 'float')\n",
    "\n",
    "            loss = tf.contrib.seq2seq.sequence_loss(logits, y[gpu_idx], float_sense_mask, name=\"loss\")\n",
    "            l2_loss = l2_lambda * tf.losses.get_regularization_loss()\n",
    "\n",
    "            total_loss = tf.cond(pretrain, lambda:loss_pos, lambda:loss + loss_pos + l2_loss)\n",
    "\n",
    "            summaries.append(tf.summary.scalar(\"loss_{}\".format(gpu_idx), loss))\n",
    "            summaries.append(tf.summary.scalar(\"loss_pos_{}\".format(gpu_idx), loss_pos))\n",
    "            summaries.append(tf.summary.scalar(\"total_loss_{}\".format(gpu_idx), total_loss))\n",
    "\n",
    "\n",
    "            optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "            grads_vars = optimizer.compute_gradients(total_loss)\n",
    "\n",
    "            clipped_grads = grads_vars\n",
    "            if(clipping == True):\n",
    "                clipped_grads = [(tf.clip_by_norm(grad, clip_norm), var) for grad, var in clipped_grads]\n",
    "\n",
    "            tower_grads.append(clipped_grads)\n",
    "            losses.append(total_loss)\n",
    "\n",
    "with tf.device('/gpu:0'):\n",
    "    tower_grads = average_gradients(tower_grads)\n",
    "    losses = tf.add_n(losses)/len(losses)\n",
    "    apply_grad_op = optimizer.apply_gradients(tower_grads, global_step=global_step)\n",
    "    summaries.append(tf.summary.scalar('total_loss', losses))\n",
    "    summaries.append(tf.summary.scalar('learning_rate', learning_rate))\n",
    "\n",
    "    variable_averages = tf.train.ExponentialMovingAverage(moving_avg_deacy, global_step)\n",
    "    variables_averages_op = variable_averages.apply(tf.trainable_variables())\n",
    "\n",
    "    train_op = tf.group(apply_grad_op, variables_averages_op)\n",
    "    saver = tf.train.Saver(tf.global_variables())\n",
    "    summary = tf.summary.merge(summaries)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.environ[\"CUDA_DEVICE_ORDER\"]=\"PCI_BUS_ID\"   # see issue #152\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0,1\"\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth = True\n",
    "config.allow_soft_placement = True\n",
    "sess = tf.Session(config=config)\n",
    "sess.run(tf.global_variables_initializer())                          # For initializing all the variables\n",
    "summary_writer = tf.summary.FileWriter(log_dir, sess.graph)          # For writing Summaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_period = 100\n",
    "log_period = 100\n",
    "\n",
    "def model(xx, yy, yy_pos, mask, smask, train_cond=True, pretrain_cond=False):\n",
    "    num_batches = int(len(xx)/(batch_size*num_gpus))\n",
    "    _losses = 0\n",
    "    temp_loss = 0\n",
    "    preds_sense = []\n",
    "    true_sense = []\n",
    "    preds_pos = []\n",
    "    true_pos = []\n",
    "    \n",
    "    for j in range(num_batches): \n",
    "        \n",
    "        s = j * batch_size * num_gpus\n",
    "        e = (j+1) * batch_size * num_gpus\n",
    "        xx_re = xx[s:e].reshape([num_gpus, batch_size, -1])\n",
    "        yy_re = yy[s:e].reshape([num_gpus, batch_size, -1])\n",
    "        yy_pos_re = yy_pos[s:e].reshape([num_gpus, batch_size, -1])\n",
    "        mask_re = mask[s:e].reshape([num_gpus, batch_size, -1])\n",
    "        smask_re = smask[s:e].reshape([num_gpus, batch_size, -1])\n",
    " \n",
    "        feed_dict = {x:xx_re, y:yy_re, y_pos:yy_pos_re, x_mask:mask_re, sense_mask:smask_re, pretrain:pretrain_cond, is_train:train_cond, input_keep_prob:keep_prob, word_emb_mat:word_embedding}\n",
    "        \n",
    "        if(train_cond==True):\n",
    "            _, _loss, step, _summary = sess.run([train_op, losses, global_step, summary], feed_dict)\n",
    "            summary_writer.add_summary(_summary, step)\n",
    "            \n",
    "            temp_loss += _loss\n",
    "            if((j+1)%log_period==0):\n",
    "                print(\"Steps: {}\".format(step), \"Loss:{0:.4f}\".format(temp_loss/log_period), \", Current Loss: {0:.4f}\".format(_loss))\n",
    "                temp_loss = 0\n",
    "            if((j+1)%save_period==0):\n",
    "                saver.save(sess, save_path=save_dir)                         \n",
    "                \n",
    "        else:\n",
    "            _loss, pred, crf_logits, trans_params_ = sess.run([total_loss, predictions, predictions_pos, total_trans_params], feed_dict)\n",
    "            \n",
    "            for i in range(num_gpus):\n",
    "                preds_sense.append(pred[i][smask_re[i]])\n",
    "                true_sense.append(yy_re[i][smask_re[i]])\n",
    "                true_pos.append(yy_pos_re[i][mask_re[i]])\n",
    "                temp = []\n",
    "                for k in range(batch_size):\n",
    "                    logit_ = crf_logits[i][k][:sum(mask_re[i][k])] # keep only the valid steps\n",
    "                    viterbi_seq, viterbi_score = tf.contrib.crf.viterbi_decode(logit_, trans_params_[i])\n",
    "                    temp += viterbi_seq\n",
    "                preds_pos.append(temp)\n",
    "\n",
    "        _losses +=_loss\n",
    "\n",
    "    if(train_cond==False): \n",
    "        sense_preds = []\n",
    "        sense_true = []\n",
    "        pos_preds = []\n",
    "        pos_true = []\n",
    "        \n",
    "        for preds in preds_sense:\n",
    "            for ps in preds:      \n",
    "                sense_preds.append(ps)  \n",
    "        for trues in true_sense:\n",
    "            for ts in trues:\n",
    "                sense_true.append(ts)\n",
    "        \n",
    "        for preds in preds_pos:\n",
    "            for ps in preds:      \n",
    "                pos_preds.append(ps)      \n",
    "        for trues in true_pos:\n",
    "            for ts in trues:\n",
    "                pos_true.append(ts)\n",
    "                \n",
    "        return _losses/num_batches, sense_preds, sense_true, pos_preds, pos_true\n",
    "\n",
    "    return _losses/num_batches, step\n",
    "\n",
    "def eval_score(yy, pred, yy_pos, pred_pos):\n",
    "    f1 = f1_score(yy, pred, average='macro')\n",
    "    accu = accuracy_score(yy, pred)\n",
    "    f1_pos = f1_score(yy_pos, pred_pos, average='macro')\n",
    "    accu_pos = accuracy_score(yy_pos, pred_pos)\n",
    "    return f1*100, accu*100, f1_pos*100, accu_pos*100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_id_train = train_data['x']\n",
    "mask_train = train_data['x_mask']\n",
    "sense_mask_train = train_data['sense_mask']\n",
    "y_train = train_data['y']\n",
    "y_pos_train = train_data['pos']\n",
    "\n",
    "x_id_val = val_data['x']\n",
    "mask_val = val_data['x_mask']\n",
    "sense_mask_val = val_data['sense_mask']\n",
    "y_val = val_data['y']\n",
    "y_pos_val = val_data['pos']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def testing():\n",
    "    start_time = time.time()\n",
    "    val_loss, val_pred, val_true, val_pred_pos, val_true_pos = model(x_id_val, y_val, y_pos_val, mask_val, sense_mask_val, train_cond=False)        \n",
    "    f1_, accu_, f1_pos_, accu_pos_ = eval_score(val_true, val_pred, val_true_pos, val_pred_pos)\n",
    "    time_taken = time.time() - start_time\n",
    "    print(\"Val: F1 Score:{0:.2f}\".format(f1_), \"Accuracy:{0:.2f}\".format(accu_), \" POS: F1 Score:{0:.2f}\".format(f1_pos_), \"Accuracy:{0:.2f}\".format(accu_pos_), \"Loss:{0:.4f}\".format(val_loss), \", Time: {0:.1f}\".format(time_taken))\n",
    "    return f1_, accu_, f1_pos_, accu_pos_\n",
    "\n",
    "def training(current_epoch, pre_train_cond):\n",
    "        random = np.random.choice(len(y_train), size=(len(y_train)), replace=False)\n",
    "        x_id_train_tmp = x_id_train[random]\n",
    "        y_train_tmp = y_train[random]\n",
    "        mask_train_tmp = mask_train[random]    \n",
    "        sense_mask_train_tmp = sense_mask_train[random]\n",
    "        y_pos_train_tmp = y_pos_train[random]\n",
    "\n",
    "        start_time = time.time()\n",
    "        train_loss, step = model(x_id_train_tmp, y_train_tmp, y_pos_train_tmp, mask_train_tmp, sense_mask_train_tmp, pretrain_cond=pre_train_cond)\n",
    "        time_taken = time.time() - start_time\n",
    "        print(\"Epoch: {}\".format(current_epoch+1),\", Step: {}\".format(step), \", loss: {0:.4f}\".format(train_loss), \", Time: {0:.1f}\".format(time_taken))\n",
    "        saver.save(sess, save_path=save_dir)                         \n",
    "        print(\"Model Saved\")\n",
    "        return [step, train_loss]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Steps: 100 Loss:2.7530 , Current Loss: 1.3150\n",
      "Steps: 200 Loss:1.0997 , Current Loss: 0.8834\n",
      "Steps: 300 Loss:0.8495 , Current Loss: 0.7899\n",
      "Steps: 400 Loss:0.7236 , Current Loss: 0.6715\n",
      "Steps: 500 Loss:0.6522 , Current Loss: 0.6031\n",
      "Steps: 600 Loss:0.6055 , Current Loss: 0.6011\n",
      "Steps: 700 Loss:0.5653 , Current Loss: 0.5640\n",
      "Steps: 800 Loss:0.5363 , Current Loss: 0.5527\n",
      "Epoch: 1 , Step: 885 , loss: 0.9292 , Time: 6593.3\n",
      "Model Saved\n",
      "Steps: 985 Loss:0.4982 , Current Loss: 0.5113\n",
      "Steps: 1085 Loss:0.4793 , Current Loss: 0.4804\n",
      "Steps: 1185 Loss:0.4628 , Current Loss: 0.4392\n",
      "Steps: 1285 Loss:0.4512 , Current Loss: 0.4618\n",
      "Steps: 1385 Loss:0.4417 , Current Loss: 0.4541\n",
      "Steps: 1485 Loss:0.4313 , Current Loss: 0.4190\n",
      "Steps: 1585 Loss:0.4195 , Current Loss: 0.4209\n",
      "Steps: 1685 Loss:0.4135 , Current Loss: 0.4002\n",
      "Epoch: 2 , Step: 1770 , loss: 0.4453 , Time: 6481.8\n",
      "Model Saved\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/users/btech/aviraj/envs/lib/python3.5/site-packages/sklearn/metrics/classification.py:1137: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val: F1 Score:1.30 Accuracy:1.98  POS: F1 Score:87.26 Accuracy:92.35 Loss:6.5045 , Time: 1338.7\n",
      "Steps: 1870 Loss:3.0857 , Current Loss: 2.7427\n",
      "Steps: 1970 Loss:2.7031 , Current Loss: 2.5829\n",
      "Steps: 2070 Loss:2.6134 , Current Loss: 2.5436\n",
      "Steps: 2170 Loss:2.5458 , Current Loss: 2.5251\n",
      "Steps: 2270 Loss:2.5081 , Current Loss: 2.6032\n",
      "Steps: 2370 Loss:2.4721 , Current Loss: 2.3927\n",
      "Steps: 2470 Loss:2.4513 , Current Loss: 2.4687\n",
      "Steps: 2570 Loss:2.4191 , Current Loss: 2.4182\n",
      "Epoch: 1 , Step: 2655 , loss: 2.5810 , Time: 6424.4\n",
      "Model Saved\n",
      "Steps: 2755 Loss:2.3788 , Current Loss: 2.3777\n",
      "Steps: 2855 Loss:2.3645 , Current Loss: 2.3862\n",
      "Steps: 2955 Loss:2.3436 , Current Loss: 2.3440\n",
      "Steps: 3055 Loss:2.3291 , Current Loss: 2.3917\n",
      "Steps: 3155 Loss:2.3128 , Current Loss: 2.3305\n",
      "Steps: 3255 Loss:2.3004 , Current Loss: 2.3307\n",
      "Steps: 3355 Loss:2.2989 , Current Loss: 2.2376\n",
      "Steps: 3455 Loss:2.2747 , Current Loss: 2.3122\n",
      "Epoch: 2 , Step: 3540 , loss: 2.3196 , Time: 6384.8\n",
      "Model Saved\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/users/btech/aviraj/envs/lib/python3.5/site-packages/sklearn/metrics/classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val: F1 Score:30.57 Accuracy:43.43  POS: F1 Score:84.94 Accuracy:91.65 Loss:2.2736 , Time: 1334.1\n",
      "Steps: 3640 Loss:2.2597 , Current Loss: 2.2908\n",
      "Steps: 3740 Loss:2.2459 , Current Loss: 2.2992\n",
      "Steps: 3940 Loss:2.2260 , Current Loss: 2.2485\n",
      "Steps: 4040 Loss:2.2197 , Current Loss: 2.1538\n",
      "Steps: 4140 Loss:2.2151 , Current Loss: 2.2150\n",
      "Steps: 4240 Loss:2.2052 , Current Loss: 2.2117\n",
      "Steps: 4340 Loss:2.1925 , Current Loss: 2.2674\n",
      "Epoch: 3 , Step: 4425 , loss: 2.2232 , Time: 6403.8\n",
      "Model Saved\n",
      "Steps: 4525 Loss:2.1854 , Current Loss: 2.1272\n",
      "Steps: 4625 Loss:2.1682 , Current Loss: 2.2396\n",
      "Steps: 4725 Loss:2.1652 , Current Loss: 2.1550\n",
      "Steps: 4825 Loss:2.1640 , Current Loss: 2.1435\n",
      "Steps: 4925 Loss:2.1604 , Current Loss: 2.0998\n",
      "Steps: 5025 Loss:2.1477 , Current Loss: 2.1546\n",
      "Steps: 5125 Loss:2.1531 , Current Loss: 2.1015\n",
      "Steps: 5225 Loss:2.1476 , Current Loss: 2.1499\n",
      "Epoch: 4 , Step: 5310 , loss: 2.1595 , Time: 6392.6\n",
      "Model Saved\n",
      "Val: F1 Score:34.55 Accuracy:46.59  POS: F1 Score:85.45 Accuracy:92.03 Loss:2.1500 , Time: 1328.5\n",
      "Steps: 5410 Loss:2.1341 , Current Loss: 2.1122\n",
      "Steps: 5510 Loss:2.1254 , Current Loss: 2.1217\n",
      "Steps: 5610 Loss:2.1078 , Current Loss: 2.1582\n",
      "Steps: 5710 Loss:2.1259 , Current Loss: 2.0772\n",
      "Steps: 5810 Loss:2.1077 , Current Loss: 2.1310\n",
      "Steps: 5910 Loss:2.1068 , Current Loss: 2.0401\n",
      "Steps: 6010 Loss:2.1020 , Current Loss: 1.9967\n",
      "Steps: 6110 Loss:2.1036 , Current Loss: 2.0884\n",
      "Epoch: 5 , Step: 6195 , loss: 2.1127 , Time: 6392.9\n",
      "Model Saved\n",
      "Steps: 6295 Loss:2.0835 , Current Loss: 1.9944\n",
      "Steps: 6395 Loss:2.0906 , Current Loss: 2.1595\n",
      "Steps: 6495 Loss:2.0769 , Current Loss: 2.1487\n",
      "Steps: 6595 Loss:2.0788 , Current Loss: 2.1296\n",
      "Steps: 6695 Loss:2.0779 , Current Loss: 2.0412\n",
      "Steps: 6795 Loss:2.0707 , Current Loss: 1.9520\n",
      "Steps: 6895 Loss:2.0608 , Current Loss: 2.0652\n",
      "Steps: 6995 Loss:2.0615 , Current Loss: 2.1730\n",
      "Epoch: 6 , Step: 7080 , loss: 2.0737 , Time: 6381.8\n",
      "Model Saved\n",
      "Val: F1 Score:37.12 Accuracy:48.25  POS: F1 Score:85.83 Accuracy:92.29 Loss:2.0685 , Time: 1327.5\n",
      "Steps: 7180 Loss:2.0449 , Current Loss: 2.0264\n",
      "Steps: 7280 Loss:2.0555 , Current Loss: 1.9954\n",
      "Steps: 7380 Loss:2.0445 , Current Loss: 2.0369\n",
      "Steps: 7480 Loss:2.0484 , Current Loss: 2.1198\n",
      "Steps: 7580 Loss:2.0463 , Current Loss: 2.0499\n",
      "Steps: 7680 Loss:2.0470 , Current Loss: 2.0310\n",
      "Steps: 7780 Loss:2.0374 , Current Loss: 2.0103\n",
      "Steps: 7880 Loss:2.0336 , Current Loss: 2.0351\n",
      "Epoch: 7 , Step: 7965 , loss: 2.0434 , Time: 6399.1\n",
      "Model Saved\n",
      "Steps: 8065 Loss:2.0227 , Current Loss: 2.0066\n",
      "Steps: 8165 Loss:2.0247 , Current Loss: 2.0645\n",
      "Steps: 8265 Loss:2.0233 , Current Loss: 1.9865\n",
      "Steps: 8365 Loss:2.0085 , Current Loss: 2.0752\n",
      "Steps: 8465 Loss:2.0132 , Current Loss: 2.0885\n",
      "Steps: 8565 Loss:2.0111 , Current Loss: 1.9952\n",
      "Steps: 8665 Loss:2.0028 , Current Loss: 1.9663\n",
      "Steps: 8765 Loss:2.0018 , Current Loss: 1.9029\n",
      "Epoch: 8 , Step: 8850 , loss: 2.0123 , Time: 6841.5\n",
      "Model Saved\n",
      "Val: F1 Score:39.50 Accuracy:49.70  POS: F1 Score:85.83 Accuracy:92.50 Loss:2.0151 , Time: 1329.8\n",
      "Steps: 8950 Loss:1.9972 , Current Loss: 2.0489\n",
      "Steps: 9050 Loss:2.0071 , Current Loss: 1.9245\n",
      "Steps: 9150 Loss:1.9982 , Current Loss: 1.9430\n",
      "Steps: 9250 Loss:1.9875 , Current Loss: 2.0231\n",
      "Steps: 9350 Loss:1.9861 , Current Loss: 1.9410\n",
      "Steps: 9450 Loss:1.9866 , Current Loss: 2.0304\n",
      "Steps: 9550 Loss:1.9783 , Current Loss: 1.9239\n",
      "Steps: 9650 Loss:1.9817 , Current Loss: 1.9716\n",
      "Epoch: 9 , Step: 9735 , loss: 1.9898 , Time: 6410.7\n",
      "Model Saved\n",
      "Steps: 9835 Loss:1.9669 , Current Loss: 2.1009\n",
      "Steps: 9935 Loss:1.9739 , Current Loss: 1.9356\n",
      "Steps: 10035 Loss:1.9690 , Current Loss: 1.9522\n",
      "Steps: 10135 Loss:1.9696 , Current Loss: 1.9704\n",
      "Steps: 10235 Loss:1.9645 , Current Loss: 1.9692\n",
      "Steps: 10335 Loss:1.9741 , Current Loss: 1.8623\n",
      "Steps: 10435 Loss:1.9617 , Current Loss: 2.0275\n",
      "Steps: 10535 Loss:1.9557 , Current Loss: 1.9592\n",
      "Epoch: 10 , Step: 10620 , loss: 1.9658 , Time: 6400.0\n",
      "Model Saved\n",
      "Val: F1 Score:41.51 Accuracy:50.98  POS: F1 Score:86.23 Accuracy:92.64 Loss:1.9669 , Time: 1459.9\n"
     ]
    }
   ],
   "source": [
    "loss_collection = []\n",
    "val_collection = []\n",
    "num_epochs = 20\n",
    "val_period = 2\n",
    "\n",
    "# Pretraining POS Tags\n",
    "training(0, True)\n",
    "training(1, True)\n",
    "testing()\n",
    "\n",
    "for i in range(num_epochs):\n",
    "    loss_collection.append(training(i, False))\n",
    "    if((i+1)%val_period==0):\n",
    "        val_collection.append(testing())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Steps: 11920 Loss:1.9396 , Current Loss: 1.9192\n",
      "Steps: 12020 Loss:1.9274 , Current Loss: 2.0037\n",
      "Steps: 12120 Loss:1.9368 , Current Loss: 1.9705\n",
      "Steps: 12220 Loss:1.9159 , Current Loss: 1.9314\n",
      "Steps: 12320 Loss:1.9359 , Current Loss: 1.8787\n",
      "Steps: 12420 Loss:1.9194 , Current Loss: 1.9753\n",
      "Steps: 12520 Loss:1.9213 , Current Loss: 1.9960\n",
      "Steps: 12620 Loss:1.9280 , Current Loss: 1.8762\n",
      "Epoch: 1 , Step: 12705 , loss: 1.9274 , Time: 8243.8\n",
      "Model Saved\n",
      "Steps: 12805 Loss:1.9194 , Current Loss: 1.9295\n",
      "Steps: 12905 Loss:1.9119 , Current Loss: 1.9011\n",
      "Steps: 13005 Loss:1.9154 , Current Loss: 1.8299\n",
      "Steps: 13105 Loss:1.9119 , Current Loss: 1.8779\n",
      "Steps: 13205 Loss:1.9129 , Current Loss: 1.8713\n",
      "Steps: 13305 Loss:1.9084 , Current Loss: 1.9974\n",
      "Steps: 13405 Loss:1.9120 , Current Loss: 1.9565\n",
      "Steps: 13505 Loss:1.9055 , Current Loss: 1.8886\n",
      "Epoch: 2 , Step: 13590 , loss: 1.9118 , Time: 9350.0\n",
      "Model Saved\n",
      "Val: F1 Score:43.30 Accuracy:51.93  POS: F1 Score:86.61 Accuracy:92.86 Loss:1.9281 , Time: 2102.4\n",
      "Steps: 13690 Loss:1.9084 , Current Loss: 1.8496\n"
     ]
    }
   ],
   "source": [
    "loss_collection = []\n",
    "val_collection = []\n",
    "num_epochs = 20\n",
    "val_period = 2\n",
    "\n",
    "for i in range(num_epochs):\n",
    "    loss_collection.append(training(i, False))\n",
    "    if((i+1)%val_period==0):\n",
    "        val_collection.append(testing())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Steps: 14590 Loss:1.8898 , Current Loss: 1.8106\n",
      "Steps: 14690 Loss:1.8922 , Current Loss: 1.9134\n",
      "Steps: 14790 Loss:1.8830 , Current Loss: 1.9143\n",
      "Steps: 14890 Loss:1.8821 , Current Loss: 1.9095\n",
      "Steps: 14990 Loss:1.8854 , Current Loss: 1.8719\n",
      "Steps: 15090 Loss:1.8841 , Current Loss: 1.9275\n",
      "Steps: 15190 Loss:1.8855 , Current Loss: 1.9128\n",
      "Steps: 15290 Loss:1.8766 , Current Loss: 1.9220\n",
      "Epoch: 1 , Step: 15375 , loss: 1.8842 , Time: 10384.0\n",
      "Model Saved\n",
      "Steps: 15475 Loss:1.8735 , Current Loss: 1.7995\n",
      "Steps: 15575 Loss:1.8888 , Current Loss: 1.7857\n",
      "Steps: 15675 Loss:1.8689 , Current Loss: 1.8067\n",
      "Steps: 15775 Loss:1.8728 , Current Loss: 1.8233\n",
      "Steps: 15875 Loss:1.8789 , Current Loss: 1.9815\n",
      "Steps: 15975 Loss:1.8744 , Current Loss: 1.8223\n",
      "Steps: 16075 Loss:1.8705 , Current Loss: 1.8704\n",
      "Steps: 16175 Loss:1.8795 , Current Loss: 1.8332\n",
      "Epoch: 2 , Step: 16260 , loss: 1.8753 , Time: 8197.5\n",
      "Model Saved\n",
      "Val: F1 Score:43.75 Accuracy:52.86  POS: F1 Score:86.77 Accuracy:92.98 Loss:1.8804 , Time: 1568.8\n",
      "Steps: 16360 Loss:1.8642 , Current Loss: 1.8766\n",
      "Steps: 16460 Loss:1.8593 , Current Loss: 1.8336\n",
      "Steps: 16560 Loss:1.8730 , Current Loss: 1.9217\n",
      "Steps: 16660 Loss:1.8626 , Current Loss: 1.8360\n",
      "Steps: 16760 Loss:1.8626 , Current Loss: 1.8052\n",
      "Steps: 16860 Loss:1.8663 , Current Loss: 1.8058\n",
      "Steps: 16960 Loss:1.8516 , Current Loss: 1.7824\n",
      "Steps: 17060 Loss:1.8692 , Current Loss: 1.9025\n",
      "Epoch: 3 , Step: 17145 , loss: 1.8624 , Time: 6578.5\n",
      "Model Saved\n",
      "Steps: 17245 Loss:1.8574 , Current Loss: 1.9363\n",
      "Steps: 17345 Loss:1.8616 , Current Loss: 1.8429\n",
      "Steps: 17445 Loss:1.8555 , Current Loss: 1.8861\n",
      "Steps: 17545 Loss:1.8563 , Current Loss: 1.9397\n",
      "Steps: 17645 Loss:1.8566 , Current Loss: 1.9174\n",
      "Steps: 17745 Loss:1.8434 , Current Loss: 1.8150\n",
      "Steps: 17845 Loss:1.8539 , Current Loss: 1.9456\n",
      "Steps: 17945 Loss:1.8590 , Current Loss: 1.7533\n",
      "Epoch: 4 , Step: 18030 , loss: 1.8557 , Time: 6498.5\n",
      "Model Saved\n",
      "Val: F1 Score:45.25 Accuracy:53.39  POS: F1 Score:86.65 Accuracy:93.05 Loss:1.8694 , Time: 1340.8\n",
      "Steps: 18130 Loss:1.8496 , Current Loss: 1.8736\n",
      "Steps: 18230 Loss:1.8418 , Current Loss: 1.8061\n",
      "Steps: 18330 Loss:1.8416 , Current Loss: 1.7851\n",
      "Steps: 18430 Loss:1.8495 , Current Loss: 1.9311\n",
      "Steps: 18530 Loss:1.8417 , Current Loss: 1.8368\n",
      "Steps: 18630 Loss:1.8403 , Current Loss: 1.8252\n",
      "Steps: 18730 Loss:1.8426 , Current Loss: 1.7870\n",
      "Steps: 18830 Loss:1.8452 , Current Loss: 1.8796\n",
      "Epoch: 5 , Step: 18915 , loss: 1.8434 , Time: 9945.3\n",
      "Model Saved\n",
      "Steps: 19015 Loss:1.8495 , Current Loss: 1.9335\n",
      "Steps: 19115 Loss:1.8378 , Current Loss: 1.8681\n",
      "Steps: 19215 Loss:1.8437 , Current Loss: 1.8594\n",
      "Steps: 19315 Loss:1.8249 , Current Loss: 1.8176\n",
      "Steps: 19415 Loss:1.8262 , Current Loss: 1.8345\n",
      "Steps: 19515 Loss:1.8287 , Current Loss: 1.8290\n",
      "Steps: 19615 Loss:1.8334 , Current Loss: 1.8861\n",
      "Steps: 19715 Loss:1.8377 , Current Loss: 1.8059\n",
      "Epoch: 6 , Step: 19800 , loss: 1.8351 , Time: 9181.9\n",
      "Model Saved\n",
      "Val: F1 Score:45.71 Accuracy:53.97  POS: F1 Score:86.92 Accuracy:93.12 Loss:1.8392 , Time: 1746.7\n",
      "Steps: 19900 Loss:1.8251 , Current Loss: 1.7824\n",
      "Steps: 20000 Loss:1.8243 , Current Loss: 1.9291\n",
      "Steps: 20100 Loss:1.8310 , Current Loss: 1.8379\n",
      "Steps: 20200 Loss:1.8189 , Current Loss: 1.7306\n",
      "Steps: 20300 Loss:1.8319 , Current Loss: 1.8149\n",
      "Steps: 20400 Loss:1.8202 , Current Loss: 1.8432\n",
      "Steps: 20500 Loss:1.8279 , Current Loss: 1.7857\n",
      "Steps: 20600 Loss:1.8216 , Current Loss: 1.8431\n",
      "Epoch: 7 , Step: 20685 , loss: 1.8253 , Time: 7882.0\n",
      "Model Saved\n",
      "Steps: 20785 Loss:1.8175 , Current Loss: 1.7888\n",
      "Steps: 20885 Loss:1.8186 , Current Loss: 1.8730\n",
      "Steps: 20985 Loss:1.8195 , Current Loss: 1.7983\n",
      "Steps: 21085 Loss:1.8229 , Current Loss: 1.8117\n",
      "Steps: 21185 Loss:1.8210 , Current Loss: 1.8836\n",
      "Steps: 21285 Loss:1.8166 , Current Loss: 1.8145\n",
      "Steps: 21385 Loss:1.8175 , Current Loss: 1.8286\n",
      "Steps: 21485 Loss:1.8184 , Current Loss: 1.8149\n",
      "Epoch: 8 , Step: 21570 , loss: 1.8189 , Time: 6487.7\n",
      "Model Saved\n",
      "Val: F1 Score:46.56 Accuracy:54.29  POS: F1 Score:87.01 Accuracy:93.18 Loss:1.8306 , Time: 1339.5\n",
      "Steps: 21670 Loss:1.8150 , Current Loss: 1.7859\n",
      "Steps: 21770 Loss:1.8185 , Current Loss: 1.7555\n",
      "Steps: 21870 Loss:1.8180 , Current Loss: 1.7819\n",
      "Steps: 21970 Loss:1.8150 , Current Loss: 1.8119\n",
      "Steps: 22070 Loss:1.8088 , Current Loss: 1.9019\n",
      "Steps: 22170 Loss:1.8059 , Current Loss: 1.8413\n",
      "Steps: 22270 Loss:1.8109 , Current Loss: 1.7009\n",
      "Steps: 22370 Loss:1.7991 , Current Loss: 1.7395\n",
      "Epoch: 9 , Step: 22455 , loss: 1.8109 , Time: 6439.2\n",
      "Model Saved\n",
      "Steps: 22555 Loss:1.8071 , Current Loss: 1.8280\n",
      "Steps: 22655 Loss:1.8024 , Current Loss: 1.8434\n",
      "Steps: 22755 Loss:1.8021 , Current Loss: 1.7447\n",
      "Steps: 22855 Loss:1.8086 , Current Loss: 1.7641\n",
      "Steps: 22955 Loss:1.8097 , Current Loss: 1.7668\n",
      "Steps: 23055 Loss:1.8013 , Current Loss: 1.7734\n",
      "Steps: 23155 Loss:1.8020 , Current Loss: 1.8109\n",
      "Steps: 23255 Loss:1.8045 , Current Loss: 1.7769\n",
      "Epoch: 10 , Step: 23340 , loss: 1.8042 , Time: 6687.3\n",
      "Model Saved\n",
      "Val: F1 Score:46.38 Accuracy:54.25  POS: F1 Score:87.11 Accuracy:93.23 Loss:1.8242 , Time: 1350.0\n",
      "Steps: 23440 Loss:1.7985 , Current Loss: 1.8842\n",
      "Steps: 23540 Loss:1.7993 , Current Loss: 1.7882\n",
      "Steps: 23640 Loss:1.7936 , Current Loss: 1.7810\n",
      "Steps: 23740 Loss:1.8028 , Current Loss: 1.7913\n",
      "Steps: 23840 Loss:1.8111 , Current Loss: 1.8512\n",
      "Steps: 23940 Loss:1.7922 , Current Loss: 1.8272\n",
      "Steps: 24040 Loss:1.7916 , Current Loss: 1.8054\n",
      "Steps: 24140 Loss:1.7931 , Current Loss: 1.6782\n",
      "Epoch: 11 , Step: 24225 , loss: 1.7979 , Time: 6445.4\n",
      "Model Saved\n",
      "Steps: 24325 Loss:1.7925 , Current Loss: 1.7538\n",
      "Steps: 24425 Loss:1.7973 , Current Loss: 1.7528\n",
      "Steps: 24525 Loss:1.7953 , Current Loss: 1.7461\n",
      "Steps: 24625 Loss:1.7895 , Current Loss: 1.8282\n",
      "Steps: 24725 Loss:1.7959 , Current Loss: 1.8065\n",
      "Steps: 24825 Loss:1.7915 , Current Loss: 1.8071\n",
      "Steps: 24925 Loss:1.7839 , Current Loss: 1.7511\n",
      "Steps: 25025 Loss:1.7997 , Current Loss: 1.7762\n",
      "Epoch: 12 , Step: 25110 , loss: 1.7928 , Time: 6443.6\n",
      "Model Saved\n",
      "Val: F1 Score:47.34 Accuracy:54.64  POS: F1 Score:87.14 Accuracy:93.28 Loss:1.8110 , Time: 1334.0\n",
      "Steps: 25210 Loss:1.7861 , Current Loss: 1.7747\n",
      "Steps: 25310 Loss:1.7861 , Current Loss: 1.7803\n",
      "Steps: 25410 Loss:1.7903 , Current Loss: 1.7616\n",
      "Steps: 25510 Loss:1.7824 , Current Loss: 1.8506\n",
      "Steps: 25610 Loss:1.7815 , Current Loss: 1.7497\n",
      "Steps: 25710 Loss:1.7743 , Current Loss: 1.8295\n",
      "Steps: 25810 Loss:1.7979 , Current Loss: 1.8360\n",
      "Steps: 25910 Loss:1.7816 , Current Loss: 1.8011\n",
      "Epoch: 13 , Step: 25995 , loss: 1.7850 , Time: 7074.1\n",
      "Model Saved\n",
      "Steps: 26095 Loss:1.7814 , Current Loss: 1.7550\n",
      "Steps: 26195 Loss:1.7850 , Current Loss: 1.8889\n",
      "Steps: 26295 Loss:1.7867 , Current Loss: 1.7959\n",
      "Steps: 26395 Loss:1.7719 , Current Loss: 1.7058\n",
      "Steps: 26495 Loss:1.7821 , Current Loss: 1.7406\n",
      "Steps: 26595 Loss:1.7841 , Current Loss: 1.8149\n",
      "Steps: 26695 Loss:1.7780 , Current Loss: 1.7717\n",
      "Steps: 26795 Loss:1.7735 , Current Loss: 1.7353\n",
      "Epoch: 14 , Step: 26880 , loss: 1.7800 , Time: 7094.6\n",
      "Model Saved\n",
      "Val: F1 Score:47.66 Accuracy:54.98  POS: F1 Score:87.26 Accuracy:93.34 Loss:1.7872 , Time: 1387.4\n",
      "Steps: 26980 Loss:1.7800 , Current Loss: 1.7911\n",
      "Steps: 27080 Loss:1.7761 , Current Loss: 1.7988\n",
      "Steps: 27180 Loss:1.7726 , Current Loss: 1.7519\n",
      "Steps: 27280 Loss:1.7634 , Current Loss: 1.7320\n",
      "Steps: 27380 Loss:1.7739 , Current Loss: 1.7650\n",
      "Steps: 27480 Loss:1.7814 , Current Loss: 1.7587\n",
      "Steps: 27580 Loss:1.7756 , Current Loss: 1.8395\n",
      "Steps: 27680 Loss:1.7777 , Current Loss: 1.8550\n",
      "Epoch: 15 , Step: 27765 , loss: 1.7744 , Time: 6909.0\n",
      "Model Saved\n",
      "Steps: 27865 Loss:1.7755 , Current Loss: 1.7743\n",
      "Steps: 27965 Loss:1.7645 , Current Loss: 1.8413\n",
      "Steps: 28065 Loss:1.7787 , Current Loss: 1.7219\n",
      "Steps: 28165 Loss:1.7615 , Current Loss: 1.7391\n",
      "Steps: 28265 Loss:1.7630 , Current Loss: 1.7339\n",
      "Steps: 28365 Loss:1.7678 , Current Loss: 1.7627\n",
      "Steps: 28465 Loss:1.7654 , Current Loss: 1.7629\n",
      "Steps: 28565 Loss:1.7758 , Current Loss: 1.8128\n",
      "Epoch: 16 , Step: 28650 , loss: 1.7683 , Time: 6935.3\n",
      "Model Saved\n",
      "Val: F1 Score:48.05 Accuracy:55.44  POS: F1 Score:87.22 Accuracy:93.38 Loss:1.7863 , Time: 1526.5\n",
      "Steps: 28750 Loss:1.7696 , Current Loss: 1.8171\n",
      "Steps: 28850 Loss:1.7684 , Current Loss: 1.7764\n",
      "Steps: 28950 Loss:1.7638 , Current Loss: 1.7337\n",
      "Steps: 29050 Loss:1.7687 , Current Loss: 1.8079\n",
      "Steps: 29150 Loss:1.7699 , Current Loss: 1.7701\n",
      "Steps: 29250 Loss:1.7576 , Current Loss: 1.7785\n",
      "Steps: 29350 Loss:1.7595 , Current Loss: 1.7868\n",
      "Steps: 29450 Loss:1.7575 , Current Loss: 1.7295\n",
      "Epoch: 17 , Step: 29535 , loss: 1.7630 , Time: 6456.0\n",
      "Model Saved\n",
      "Steps: 29635 Loss:1.7656 , Current Loss: 1.7812\n",
      "Steps: 29735 Loss:1.7496 , Current Loss: 1.7135\n",
      "Steps: 29835 Loss:1.7584 , Current Loss: 1.8268\n",
      "Steps: 29935 Loss:1.7472 , Current Loss: 1.7764\n",
      "Steps: 30035 Loss:1.7533 , Current Loss: 1.7067\n",
      "Steps: 30135 Loss:1.7580 , Current Loss: 1.7131\n",
      "Steps: 30235 Loss:1.7713 , Current Loss: 1.7302\n",
      "Steps: 30335 Loss:1.7480 , Current Loss: 1.8137\n",
      "Epoch: 18 , Step: 30420 , loss: 1.7571 , Time: 6508.7\n",
      "Model Saved\n",
      "Val: F1 Score:48.18 Accuracy:55.41  POS: F1 Score:87.26 Accuracy:93.41 Loss:1.7786 , Time: 1334.3\n",
      "Steps: 30520 Loss:1.7563 , Current Loss: 1.7713\n",
      "Steps: 30620 Loss:1.7510 , Current Loss: 1.8283\n",
      "Steps: 30720 Loss:1.7556 , Current Loss: 1.7782\n",
      "Steps: 30820 Loss:1.7458 , Current Loss: 1.6993\n",
      "Steps: 30920 Loss:1.7479 , Current Loss: 1.7470\n",
      "Steps: 31020 Loss:1.7543 , Current Loss: 1.7855\n",
      "Steps: 31120 Loss:1.7541 , Current Loss: 1.7775\n",
      "Steps: 31220 Loss:1.7520 , Current Loss: 1.6989\n",
      "Epoch: 19 , Step: 31305 , loss: 1.7528 , Time: 6400.7\n",
      "Model Saved\n",
      "Steps: 31405 Loss:1.7431 , Current Loss: 1.7244\n",
      "Steps: 31505 Loss:1.7468 , Current Loss: 1.7764\n",
      "Steps: 31605 Loss:1.7539 , Current Loss: 1.7116\n",
      "Steps: 31705 Loss:1.7564 , Current Loss: 1.6907\n",
      "Steps: 31805 Loss:1.7365 , Current Loss: 1.7478\n",
      "Steps: 31905 Loss:1.7439 , Current Loss: 1.7309\n",
      "Steps: 32005 Loss:1.7489 , Current Loss: 1.7469\n",
      "Steps: 32105 Loss:1.7534 , Current Loss: 1.7625\n",
      "Epoch: 20 , Step: 32190 , loss: 1.7476 , Time: 6881.2\n",
      "Model Saved\n",
      "Val: F1 Score:48.15 Accuracy:55.71  POS: F1 Score:87.43 Accuracy:93.46 Loss:1.7688 , Time: 1352.5\n"
     ]
    }
   ],
   "source": [
    "loss_collection = []\n",
    "val_collection = []\n",
    "num_epochs = 20\n",
    "val_period = 2\n",
    "\n",
    "for i in range(num_epochs):\n",
    "    loss_collection.append(training(i, False))\n",
    "    if((i+1)%val_period==0):\n",
    "        val_collection.append(testing())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Steps: 32790 Loss:1.7450 , Current Loss: 1.7841\n",
      "Steps: 32890 Loss:1.7457 , Current Loss: 1.6578\n",
      "Steps: 32990 Loss:1.7445 , Current Loss: 1.7572\n",
      "Steps: 33090 Loss:1.7359 , Current Loss: 1.7636\n",
      "Steps: 33190 Loss:1.7379 , Current Loss: 1.7785\n",
      "Steps: 33290 Loss:1.7408 , Current Loss: 1.8216\n",
      "Steps: 33390 Loss:1.7497 , Current Loss: 1.7257\n",
      "Steps: 33490 Loss:1.7443 , Current Loss: 1.8135\n",
      "Epoch: 1 , Step: 33575 , loss: 1.7427 , Time: 10418.4\n",
      "Model Saved\n",
      "Steps: 33675 Loss:1.7355 , Current Loss: 1.6992\n",
      "Steps: 33775 Loss:1.7324 , Current Loss: 1.7632\n",
      "Steps: 33875 Loss:1.7411 , Current Loss: 1.6326\n",
      "Steps: 33975 Loss:1.7422 , Current Loss: 1.7930\n",
      "Steps: 34075 Loss:1.7450 , Current Loss: 1.7360\n",
      "Steps: 34175 Loss:1.7369 , Current Loss: 1.7717\n",
      "Steps: 34275 Loss:1.7361 , Current Loss: 1.7250\n",
      "Steps: 34375 Loss:1.7365 , Current Loss: 1.7765\n",
      "Epoch: 2 , Step: 34460 , loss: 1.7374 , Time: 7592.0\n",
      "Model Saved\n",
      "Val: F1 Score:48.27 Accuracy:55.87  POS: F1 Score:87.41 Accuracy:93.47 Loss:1.7595 , Time: 1358.4\n",
      "Steps: 34560 Loss:1.7395 , Current Loss: 1.7101\n",
      "Steps: 34660 Loss:1.7268 , Current Loss: 1.7800\n",
      "Steps: 34760 Loss:1.7394 , Current Loss: 1.7142\n",
      "Steps: 34860 Loss:1.7309 , Current Loss: 1.7311\n",
      "Steps: 34960 Loss:1.7334 , Current Loss: 1.7379\n",
      "Steps: 35060 Loss:1.7267 , Current Loss: 1.6503\n",
      "Steps: 35160 Loss:1.7279 , Current Loss: 1.7527\n",
      "Steps: 35260 Loss:1.7414 , Current Loss: 1.7796\n",
      "Epoch: 3 , Step: 35345 , loss: 1.7342 , Time: 6451.0\n",
      "Model Saved\n",
      "Steps: 35445 Loss:1.7322 , Current Loss: 1.6775\n",
      "Steps: 35545 Loss:1.7289 , Current Loss: 1.6350\n",
      "Steps: 35645 Loss:1.7286 , Current Loss: 1.7331\n",
      "Steps: 35745 Loss:1.7252 , Current Loss: 1.7926\n",
      "Steps: 35845 Loss:1.7303 , Current Loss: 1.7349\n",
      "Steps: 35945 Loss:1.7299 , Current Loss: 1.6307\n",
      "Steps: 36045 Loss:1.7255 , Current Loss: 1.8291\n",
      "Steps: 36145 Loss:1.7274 , Current Loss: 1.7349\n",
      "Epoch: 4 , Step: 36230 , loss: 1.7286 , Time: 6455.9\n",
      "Model Saved\n",
      "Val: F1 Score:49.10 Accuracy:56.04  POS: F1 Score:87.53 Accuracy:93.52 Loss:1.7539 , Time: 1342.8\n",
      "Steps: 36330 Loss:1.7274 , Current Loss: 1.6486\n",
      "Steps: 36430 Loss:1.7262 , Current Loss: 1.7175\n",
      "Steps: 36530 Loss:1.7336 , Current Loss: 1.7625\n",
      "Steps: 36630 Loss:1.7185 , Current Loss: 1.7575\n",
      "Steps: 36730 Loss:1.7217 , Current Loss: 1.7972\n",
      "Steps: 36830 Loss:1.7315 , Current Loss: 1.6372\n",
      "Steps: 36930 Loss:1.7195 , Current Loss: 1.6436\n",
      "Steps: 37030 Loss:1.7316 , Current Loss: 1.8047\n",
      "Epoch: 5 , Step: 37115 , loss: 1.7256 , Time: 6441.6\n",
      "Model Saved\n",
      "Steps: 37215 Loss:1.7166 , Current Loss: 1.6087\n",
      "Steps: 37315 Loss:1.7177 , Current Loss: 1.6355\n",
      "Steps: 37415 Loss:1.7267 , Current Loss: 1.7267\n",
      "Steps: 37515 Loss:1.7275 , Current Loss: 1.7334\n",
      "Steps: 37615 Loss:1.7187 , Current Loss: 1.6933\n",
      "Steps: 37715 Loss:1.7225 , Current Loss: 1.7574\n",
      "Steps: 37815 Loss:1.7243 , Current Loss: 1.7004\n",
      "Steps: 37915 Loss:1.7213 , Current Loss: 1.7386\n",
      "Epoch: 6 , Step: 38000 , loss: 1.7218 , Time: 8093.3\n",
      "Model Saved\n",
      "Val: F1 Score:49.12 Accuracy:56.42  POS: F1 Score:87.53 Accuracy:93.53 Loss:1.7440 , Time: 1657.1\n",
      "Steps: 38100 Loss:1.7117 , Current Loss: 1.7080\n",
      "Steps: 38200 Loss:1.7243 , Current Loss: 1.7195\n",
      "Steps: 38300 Loss:1.7179 , Current Loss: 1.6172\n",
      "Steps: 38400 Loss:1.7151 , Current Loss: 1.7564\n",
      "Steps: 38500 Loss:1.7218 , Current Loss: 1.7111\n",
      "Steps: 38600 Loss:1.7178 , Current Loss: 1.6174\n",
      "Steps: 38700 Loss:1.7209 , Current Loss: 1.6533\n",
      "Steps: 38800 Loss:1.7234 , Current Loss: 1.7100\n",
      "Epoch: 7 , Step: 38885 , loss: 1.7192 , Time: 6567.8\n",
      "Model Saved\n",
      "Steps: 38985 Loss:1.7186 , Current Loss: 1.7830\n",
      "Steps: 39085 Loss:1.7251 , Current Loss: 1.7200\n",
      "Steps: 39185 Loss:1.7127 , Current Loss: 1.7650\n",
      "Steps: 39285 Loss:1.7121 , Current Loss: 1.7443\n",
      "Steps: 39385 Loss:1.7090 , Current Loss: 1.6858\n",
      "Steps: 39485 Loss:1.7208 , Current Loss: 1.7590\n",
      "Steps: 39585 Loss:1.7206 , Current Loss: 1.6597\n",
      "Steps: 39685 Loss:1.7127 , Current Loss: 1.7404\n",
      "Epoch: 8 , Step: 39770 , loss: 1.7160 , Time: 6459.4\n",
      "Model Saved\n",
      "Val: F1 Score:49.03 Accuracy:56.50  POS: F1 Score:87.68 Accuracy:93.56 Loss:1.7329 , Time: 1340.9\n",
      "Steps: 39870 Loss:1.7108 , Current Loss: 1.7143\n",
      "Steps: 39970 Loss:1.7217 , Current Loss: 1.7446\n",
      "Steps: 40070 Loss:1.7094 , Current Loss: 1.7135\n",
      "Steps: 40170 Loss:1.7150 , Current Loss: 1.6939\n",
      "Steps: 40270 Loss:1.7180 , Current Loss: 1.7534\n",
      "Steps: 40370 Loss:1.7110 , Current Loss: 1.7177\n",
      "Steps: 40470 Loss:1.7090 , Current Loss: 1.6691\n",
      "Steps: 40570 Loss:1.7129 , Current Loss: 1.6822\n",
      "Epoch: 9 , Step: 40655 , loss: 1.7140 , Time: 6800.2\n",
      "Model Saved\n",
      "Steps: 40755 Loss:1.7158 , Current Loss: 1.6764\n",
      "Steps: 40855 Loss:1.6968 , Current Loss: 1.6868\n",
      "Steps: 40955 Loss:1.7158 , Current Loss: 1.7101\n",
      "Steps: 41055 Loss:1.7178 , Current Loss: 1.7184\n",
      "Steps: 41155 Loss:1.7127 , Current Loss: 1.7458\n",
      "Steps: 41255 Loss:1.7057 , Current Loss: 1.6824\n",
      "Steps: 41355 Loss:1.7133 , Current Loss: 1.6037\n",
      "Steps: 41455 Loss:1.7103 , Current Loss: 1.6755\n",
      "Epoch: 10 , Step: 41540 , loss: 1.7104 , Time: 9538.1\n",
      "Model Saved\n",
      "Val: F1 Score:49.55 Accuracy:56.60  POS: F1 Score:87.55 Accuracy:93.59 Loss:1.7231 , Time: 1801.8\n",
      "Steps: 41640 Loss:1.7072 , Current Loss: 1.6732\n",
      "Steps: 41740 Loss:1.7025 , Current Loss: 1.6124\n",
      "Steps: 41840 Loss:1.7108 , Current Loss: 1.7237\n",
      "Steps: 41940 Loss:1.7045 , Current Loss: 1.7481\n",
      "Steps: 42040 Loss:1.7127 , Current Loss: 1.7408\n",
      "Steps: 42140 Loss:1.7043 , Current Loss: 1.7281\n",
      "Steps: 42240 Loss:1.7123 , Current Loss: 1.6586\n",
      "Steps: 42340 Loss:1.7120 , Current Loss: 1.6788\n",
      "Epoch: 11 , Step: 42425 , loss: 1.7076 , Time: 8035.1\n",
      "Model Saved\n",
      "Steps: 42525 Loss:1.7100 , Current Loss: 1.6099\n",
      "Steps: 42625 Loss:1.7018 , Current Loss: 1.6689\n",
      "Steps: 42725 Loss:1.6992 , Current Loss: 1.7200\n",
      "Steps: 42825 Loss:1.6972 , Current Loss: 1.6442\n",
      "Steps: 42925 Loss:1.6975 , Current Loss: 1.7315\n",
      "Steps: 43025 Loss:1.7140 , Current Loss: 1.6827\n",
      "Steps: 43125 Loss:1.7063 , Current Loss: 1.7392\n",
      "Steps: 43225 Loss:1.7066 , Current Loss: 1.7379\n",
      "Epoch: 12 , Step: 43310 , loss: 1.7033 , Time: 9646.0\n",
      "Model Saved\n",
      "Val: F1 Score:50.03 Accuracy:56.79  POS: F1 Score:87.65 Accuracy:93.63 Loss:1.7265 , Time: 2115.6\n",
      "Steps: 43410 Loss:1.6913 , Current Loss: 1.6984\n",
      "Steps: 43510 Loss:1.7031 , Current Loss: 1.6560\n",
      "Steps: 43610 Loss:1.6964 , Current Loss: 1.7432\n",
      "Steps: 43710 Loss:1.7009 , Current Loss: 1.8064\n",
      "Steps: 43810 Loss:1.7013 , Current Loss: 1.6524\n",
      "Steps: 43910 Loss:1.7003 , Current Loss: 1.6452\n",
      "Steps: 44010 Loss:1.6942 , Current Loss: 1.8259\n",
      "Steps: 44110 Loss:1.6961 , Current Loss: 1.6860\n",
      "Epoch: 13 , Step: 44195 , loss: 1.6985 , Time: 9943.0\n",
      "Model Saved\n",
      "Steps: 44295 Loss:1.6920 , Current Loss: 1.7376\n",
      "Steps: 44395 Loss:1.6972 , Current Loss: 1.6580\n",
      "Steps: 44495 Loss:1.6938 , Current Loss: 1.6731\n",
      "Steps: 44595 Loss:1.6931 , Current Loss: 1.6792\n",
      "Steps: 44695 Loss:1.7056 , Current Loss: 1.6750\n",
      "Steps: 44795 Loss:1.6956 , Current Loss: 1.7344\n",
      "Steps: 44895 Loss:1.6957 , Current Loss: 1.6499\n",
      "Steps: 44995 Loss:1.6954 , Current Loss: 1.7403\n",
      "Epoch: 14 , Step: 45080 , loss: 1.6969 , Time: 10235.4\n",
      "Model Saved\n",
      "Val: F1 Score:49.36 Accuracy:56.86  POS: F1 Score:87.74 Accuracy:93.65 Loss:1.7197 , Time: 2186.3\n",
      "Steps: 45180 Loss:1.6896 , Current Loss: 1.7360\n",
      "Steps: 45280 Loss:1.7049 , Current Loss: 1.6794\n",
      "Steps: 45380 Loss:1.6955 , Current Loss: 1.7345\n",
      "Steps: 45480 Loss:1.6949 , Current Loss: 1.6547\n",
      "Steps: 45580 Loss:1.6936 , Current Loss: 1.5941\n",
      "Steps: 45680 Loss:1.6981 , Current Loss: 1.6177\n",
      "Steps: 45780 Loss:1.7008 , Current Loss: 1.7513\n",
      "Steps: 45880 Loss:1.7051 , Current Loss: 1.7650\n",
      "Epoch: 15 , Step: 45965 , loss: 1.6966 , Time: 10207.3\n",
      "Model Saved\n",
      "Steps: 46065 Loss:1.6962 , Current Loss: 1.7005\n",
      "Steps: 46165 Loss:1.6927 , Current Loss: 1.6695\n",
      "Steps: 46265 Loss:1.6876 , Current Loss: 1.6291\n",
      "Steps: 46365 Loss:1.6916 , Current Loss: 1.6755\n",
      "Steps: 46465 Loss:1.6922 , Current Loss: 1.7822\n",
      "Steps: 46565 Loss:1.6885 , Current Loss: 1.7250\n",
      "Steps: 46665 Loss:1.6960 , Current Loss: 1.7452\n",
      "Steps: 46765 Loss:1.6938 , Current Loss: 1.6246\n",
      "Epoch: 16 , Step: 46850 , loss: 1.6929 , Time: 10248.9\n",
      "Model Saved\n",
      "Val: F1 Score:49.71 Accuracy:57.01  POS: F1 Score:87.72 Accuracy:93.66 Loss:1.7139 , Time: 1952.2\n",
      "Steps: 46950 Loss:1.6904 , Current Loss: 1.7136\n",
      "Steps: 47050 Loss:1.6845 , Current Loss: 1.6140\n",
      "Steps: 47150 Loss:1.6859 , Current Loss: 1.8621\n",
      "Steps: 47250 Loss:1.6922 , Current Loss: 1.7401\n",
      "Steps: 47350 Loss:1.6956 , Current Loss: 1.6994\n",
      "Steps: 47450 Loss:1.6851 , Current Loss: 1.6145\n",
      "Steps: 47550 Loss:1.6857 , Current Loss: 1.7318\n",
      "Steps: 47650 Loss:1.6880 , Current Loss: 1.6984\n",
      "Epoch: 17 , Step: 47735 , loss: 1.6890 , Time: 8913.7\n",
      "Model Saved\n",
      "Steps: 47835 Loss:1.6802 , Current Loss: 1.5862\n",
      "Steps: 47935 Loss:1.6960 , Current Loss: 1.7680\n",
      "Steps: 48035 Loss:1.6934 , Current Loss: 1.7045\n",
      "Steps: 48135 Loss:1.6870 , Current Loss: 1.7571\n",
      "Steps: 48235 Loss:1.6877 , Current Loss: 1.6194\n",
      "Steps: 48335 Loss:1.6844 , Current Loss: 1.6885\n",
      "Steps: 48435 Loss:1.6842 , Current Loss: 1.6845\n",
      "Steps: 48535 Loss:1.6849 , Current Loss: 1.6751\n",
      "Epoch: 18 , Step: 48620 , loss: 1.6873 , Time: 6621.6\n",
      "Model Saved\n",
      "Val: F1 Score:50.23 Accuracy:57.13  POS: F1 Score:87.82 Accuracy:93.67 Loss:1.7080 , Time: 1469.7\n",
      "Steps: 48720 Loss:1.6896 , Current Loss: 1.7196\n",
      "Steps: 48820 Loss:1.6883 , Current Loss: 1.7047\n",
      "Steps: 48920 Loss:1.6861 , Current Loss: 1.6231\n",
      "Steps: 49020 Loss:1.6835 , Current Loss: 1.7334\n",
      "Steps: 49120 Loss:1.6836 , Current Loss: 1.6934\n",
      "Steps: 49220 Loss:1.6907 , Current Loss: 1.6771\n",
      "Steps: 49320 Loss:1.6862 , Current Loss: 1.7084\n",
      "Steps: 49420 Loss:1.6825 , Current Loss: 1.8298\n",
      "Epoch: 19 , Step: 49505 , loss: 1.6859 , Time: 7377.7\n",
      "Model Saved\n",
      "Steps: 49605 Loss:1.6842 , Current Loss: 1.7233\n",
      "Steps: 49705 Loss:1.6857 , Current Loss: 1.6291\n",
      "Steps: 49805 Loss:1.6763 , Current Loss: 1.7333\n",
      "Steps: 49905 Loss:1.6699 , Current Loss: 1.6770\n",
      "Steps: 50005 Loss:1.6810 , Current Loss: 1.6928\n",
      "Steps: 50105 Loss:1.6863 , Current Loss: 1.6957\n",
      "Steps: 50205 Loss:1.6872 , Current Loss: 1.7248\n",
      "Steps: 50305 Loss:1.6827 , Current Loss: 1.6191\n",
      "Epoch: 20 , Step: 50390 , loss: 1.6812 , Time: 6423.0\n",
      "Model Saved\n",
      "Val: F1 Score:50.65 Accuracy:57.15  POS: F1 Score:87.84 Accuracy:93.70 Loss:1.7033 , Time: 1271.4\n"
     ]
    }
   ],
   "source": [
    "loss_collection = []\n",
    "val_collection = []\n",
    "num_epochs = 20\n",
    "val_period = 2\n",
    "\n",
    "for i in range(num_epochs):\n",
    "    loss_collection.append(training(i, False))\n",
    "    if((i+1)%val_period==0):\n",
    "        val_collection.append(testing())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "start_time = time.time()\n",
    "train_loss, train_pred, train_true, train_pred_pos, train_true_pos = model(x_id_train, y_train, y_pos_train, mask_train, sense_mask_train, train_cond=False)        \n",
    "f1_, accu_, f1_pos_, accu_pos_ = etrain_score(train_true, train_pred, train_true_pos, train_pred_pos)\n",
    "time_taken = time.time() - start_time\n",
    "print(\"train: F1 Score:{0:.2f}\".format(f1_), \"Accuracy:{0:.2f}\".format(accu_), \" POS: F1 Score:{0:.2f}\".format(f1_pos_), \"Accuracy:{0:.2f}\".format(accu_pos_), \"Loss:{0:.4f}\".format(train_loss), \", Time: {0:.1f}\".format(time_taken))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "saver.restore(sess, save_dir)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "envs",
   "language": "python",
   "name": "cs771"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}