one_million/all-word/Model-aw-lex-local_attention-fast-v2-4.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### fast local attention \n",
    "\n",
    "#### batch size 128, learning rate = 0.001, windows size 5, num_gpus = 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "tf.logging.set_verbosity(tf.logging.WARN)\n",
    "import pickle\n",
    "import numpy as np\n",
    "import os\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import accuracy_score\n",
    "import os\n",
    "from tensorflow.python.client import device_lib\n",
    "from collections import Counter\n",
    "import time\n",
    "\n",
    "VERY_BIG_NUMBER = 1e30"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = open('../../Glove/word_embedding_glove', 'rb')\n",
    "word_embedding = pickle.load(f)\n",
    "f.close()\n",
    "\n",
    "word_embedding = word_embedding[: len(word_embedding)-1]\n",
    "\n",
    "f = open('../../Glove/vocab_glove', 'rb')\n",
    "vocab = pickle.load(f)\n",
    "f.close()\n",
    "\n",
    "word2id = dict((w, i) for i,w in enumerate(vocab))\n",
    "id2word = dict((i, w) for i,w in enumerate(vocab))\n",
    "\n",
    "unknown_token = \"UNKNOWN_TOKEN\"\n",
    "\n",
    "# Model Description\n",
    "model_name = 'model-aw-lex-local-att-fast-v2-4'\n",
    "model_dir = '../output/all-word/' + model_name\n",
    "save_dir = os.path.join(model_dir, \"save/\")\n",
    "log_dir = os.path.join(model_dir, \"log\")\n",
    "\n",
    "if not os.path.exists(model_dir):\n",
    "    os.mkdir(model_dir)\n",
    "if not os.path.exists(save_dir):\n",
    "    os.mkdir(save_dir)\n",
    "if not os.path.exists(log_dir):\n",
    "    os.mkdir(log_dir)\n",
    "\n",
    "with open('../../../dataset/train_val_data_fine/all_word_lex','rb') as f:\n",
    "    train_data, val_data = pickle.load(f)    \n",
    "    \n",
    "# Parameters\n",
    "mode = 'train'\n",
    "num_senses = 45\n",
    "num_pos = 12\n",
    "batch_size = 128\n",
    "vocab_size = len(vocab)\n",
    "unk_vocab_size = 1\n",
    "word_emb_size = len(word_embedding[0])\n",
    "max_sent_size = 200\n",
    "hidden_size = 256\n",
    "num_filter = 256\n",
    "window_size = 5\n",
    "kernel_size = 5\n",
    "keep_prob = 0.3\n",
    "l2_lambda = 0.001\n",
    "init_lr = 0.001\n",
    "decay_steps = 500\n",
    "decay_rate = 0.99\n",
    "clip_norm = 1\n",
    "clipping = True\n",
    "moving_avg_deacy = 0.999\n",
    "num_gpus = 6\n",
    "width = int(window_size/2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def average_gradients(tower_grads):\n",
    "    average_grads = []\n",
    "    for grad_and_vars in zip(*tower_grads):\n",
    "        # Note that each grad_and_vars looks like the following:\n",
    "        #   ((grad0_gpu0, var0_gpu0), ... , (grad0_gpuN, var0_gpuN))\n",
    "        grads = []\n",
    "        for g, _ in grad_and_vars:\n",
    "            # Add 0 dimension to the gradients to represent the tower.\n",
    "            expanded_g = tf.expand_dims(g, 0)\n",
    "\n",
    "            # Append on a 'tower' dimension which we will average over below.\n",
    "            grads.append(expanded_g)\n",
    "\n",
    "        # Average over the 'tower' dimension.\n",
    "        grad = tf.concat(grads, 0)\n",
    "        grad = tf.reduce_mean(grad, axis=0)\n",
    "\n",
    "        # Keep in mind that the Variables are redundant because they are shared\n",
    "        # across towers. So .. we will just return the first tower's pointer to\n",
    "        # the Variable.\n",
    "        v = grad_and_vars[0][1]\n",
    "        grad_and_var = (grad, v)\n",
    "        average_grads.append(grad_and_var)\n",
    "    return average_grads"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# MODEL\n",
    "device_num = 0\n",
    "tower_grads = []\n",
    "losses = []\n",
    "predictions = []\n",
    "predictions_pos = []\n",
    "\n",
    "x = tf.placeholder('int32', [num_gpus, batch_size, max_sent_size], name=\"x\")\n",
    "y = tf.placeholder('int32', [num_gpus, batch_size, max_sent_size], name=\"y\")\n",
    "y_pos = tf.placeholder('int32', [num_gpus, batch_size, max_sent_size], name=\"y\")\n",
    "x_mask  = tf.placeholder('bool', [num_gpus, batch_size, max_sent_size], name='x_mask') \n",
    "sense_mask  = tf.placeholder('bool', [num_gpus, batch_size, max_sent_size], name='sense_mask')\n",
    "is_train = tf.placeholder('bool', [], name='is_train')\n",
    "word_emb_mat = tf.placeholder('float', [None, word_emb_size], name='emb_mat')\n",
    "input_keep_prob = tf.cond(is_train,lambda:keep_prob, lambda:tf.constant(1.0))\n",
    "pretrain = tf.placeholder('bool', [], name=\"pretrain\")\n",
    "\n",
    "global_step = tf.Variable(0, trainable=False, name=\"global_step\")\n",
    "learning_rate = tf.train.exponential_decay(init_lr, global_step, decay_steps, decay_rate, staircase=True)\n",
    "summaries = []\n",
    "\n",
    "def global_attention(input_x, input_mask, W_att):\n",
    "    h_masked = tf.boolean_mask(input_x, input_mask)\n",
    "    h_tanh = tf.tanh(h_masked)\n",
    "    u = tf.matmul(h_tanh, W_att)\n",
    "    a = tf.nn.softmax(u)\n",
    "    c = tf.reduce_sum(tf.multiply(h_masked, a), axis=0)  \n",
    "    return c\n",
    "\n",
    "with tf.variable_scope(\"word_embedding\"):\n",
    "    unk_word_emb_mat = tf.get_variable(\"word_emb_mat\", dtype='float', shape=[unk_vocab_size, word_emb_size], initializer=tf.contrib.layers.xavier_initializer(uniform=True, seed=0, dtype=tf.float32))\n",
    "    final_word_emb_mat = tf.concat([word_emb_mat, unk_word_emb_mat], 0)\n",
    "\n",
    "with tf.variable_scope(tf.get_variable_scope()):\n",
    "    for gpu_idx in range(num_gpus):\n",
    "        if gpu_idx>=3:\n",
    "            device_num = 1\n",
    "        with tf.name_scope(\"model_{}\".format(gpu_idx)) as scope, tf.device('/gpu:%d' % device_num):\n",
    "\n",
    "            if gpu_idx > 0:\n",
    "                    tf.get_variable_scope().reuse_variables()\n",
    "\n",
    "            with tf.name_scope(\"word\"):\n",
    "                Wx = tf.nn.embedding_lookup(final_word_emb_mat, x[gpu_idx])  \n",
    "\n",
    "            float_x_mask = tf.cast(x_mask[gpu_idx], 'float')\n",
    "            x_len = tf.reduce_sum(tf.cast(x_mask[gpu_idx], 'int32'), axis=1)\n",
    "            \n",
    "            with tf.variable_scope(\"lstm1\"):\n",
    "                cell_fw1 = tf.contrib.rnn.BasicLSTMCell(hidden_size,state_is_tuple=True)\n",
    "                cell_bw1 = tf.contrib.rnn.BasicLSTMCell(hidden_size,state_is_tuple=True)\n",
    "\n",
    "                d_cell_fw1 = tf.contrib.rnn.DropoutWrapper(cell_fw1, input_keep_prob=input_keep_prob)\n",
    "                d_cell_bw1 = tf.contrib.rnn.DropoutWrapper(cell_bw1, input_keep_prob=input_keep_prob)\n",
    "\n",
    "                (fw_h1, bw_h1), _ = tf.nn.bidirectional_dynamic_rnn(d_cell_fw1, d_cell_bw1, Wx, sequence_length=x_len, dtype='float', scope='lstm1')\n",
    "                h1 = tf.concat([fw_h1, bw_h1], 2)\n",
    "\n",
    "            with tf.variable_scope(\"lstm2\"):\n",
    "                cell_fw2 = tf.contrib.rnn.BasicLSTMCell(hidden_size,state_is_tuple=True)\n",
    "                cell_bw2 = tf.contrib.rnn.BasicLSTMCell(hidden_size,state_is_tuple=True)\n",
    "\n",
    "                d_cell_fw2 = tf.contrib.rnn.DropoutWrapper(cell_fw2, input_keep_prob=input_keep_prob)\n",
    "                d_cell_bw2 = tf.contrib.rnn.DropoutWrapper(cell_bw2, input_keep_prob=input_keep_prob)\n",
    "\n",
    "                (fw_h2, bw_h2), _ = tf.nn.bidirectional_dynamic_rnn(d_cell_fw2, d_cell_bw2, h1, sequence_length=x_len, dtype='float', scope='lstm2')\n",
    "                h = tf.concat([fw_h2, bw_h2], 2)\n",
    "            \n",
    "            with tf.variable_scope(\"local_attention\"):\n",
    "                W_att_local = tf.get_variable(\"W_att_local\", shape=[2*hidden_size, 1], initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1, seed=gpu_idx*10))\n",
    "                flat_h = tf.reshape(h, [batch_size*max_sent_size, tf.shape(h)[2]])\n",
    "                h_tanh = tf.tanh(flat_h)\n",
    "                u_flat = tf.matmul(h_tanh, W_att_local)\n",
    "                u_local = tf.reshape(u_flat, [batch_size, max_sent_size])\n",
    "                final_u = (tf.cast(x_mask[gpu_idx], 'float') -1)*VERY_BIG_NUMBER + u_local\n",
    "                c_local = tf.map_fn(lambda i:tf.reduce_sum(tf.multiply(h[:, tf.maximum(0, i-width-1):tf.minimum(1+width+i, max_sent_size)],\n",
    "                    tf.expand_dims(tf.nn.softmax(final_u[:, tf.maximum(0, i-width-1):tf.minimum(1+width+i, max_sent_size)], 1), 2)), axis=1),\n",
    "                    tf.range(max_sent_size), dtype=tf.float32)  \n",
    "            \n",
    "            c_local = tf.transpose(c_local, perm=[1,0,2]) \n",
    "            h_final = tf.multiply(c_local, tf.expand_dims(float_x_mask, 2))\n",
    "            flat_h_final = tf.reshape(h_final, [-1, tf.shape(h_final)[2]])\n",
    "            with tf.variable_scope(\"hidden_layer\"):\n",
    "                W = tf.get_variable(\"W\", shape=[2*hidden_size, 2*hidden_size], initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1, seed=gpu_idx*20))\n",
    "                b = tf.get_variable(\"b\", shape=[2*hidden_size], initializer=tf.zeros_initializer())\n",
    "                drop_flat_h_final = tf.nn.dropout(flat_h_final, input_keep_prob)\n",
    "                flat_hl = tf.matmul(drop_flat_h_final, W) + b\n",
    "\n",
    "            with tf.variable_scope(\"softmax_layer\"):\n",
    "                W = tf.get_variable(\"W\", shape=[2*hidden_size, num_senses], initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1, seed=gpu_idx*20))\n",
    "                b = tf.get_variable(\"b\", shape=[num_senses], initializer=tf.zeros_initializer())\n",
    "                drop_flat_hl = tf.nn.dropout(flat_hl, input_keep_prob)\n",
    "                flat_logits_sense = tf.matmul(drop_flat_hl, W) + b\n",
    "                logits = tf.reshape(flat_logits_sense, [batch_size, max_sent_size, num_senses])\n",
    "                predictions.append(tf.argmax(logits, 2))\n",
    "\n",
    "            with tf.variable_scope(\"softmax_layer_pos\"):\n",
    "                W = tf.get_variable(\"W\", shape=[2*hidden_size, num_pos], initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1, seed=gpu_idx*30))\n",
    "                b = tf.get_variable(\"b\", shape=[num_pos], initializer=tf.zeros_initializer())\n",
    "                flat_h1 = tf.reshape(h1, [-1, tf.shape(h1)[2]])\n",
    "                drop_flat_hl = tf.nn.dropout(flat_hl, input_keep_prob)\n",
    "                flat_logits_pos = tf.matmul(drop_flat_hl, W) + b\n",
    "                logits_pos = tf.reshape(flat_logits_pos, [batch_size, max_sent_size, num_pos])\n",
    "                predictions_pos.append(tf.argmax(logits_pos, 2))\n",
    "\n",
    "\n",
    "            float_sense_mask = tf.cast(sense_mask[gpu_idx], 'float')\n",
    "\n",
    "            loss = tf.contrib.seq2seq.sequence_loss(logits, y[gpu_idx], float_sense_mask, name=\"loss\")\n",
    "            loss_pos = tf.contrib.seq2seq.sequence_loss(logits_pos, y_pos[gpu_idx], float_x_mask, name=\"loss_\")\n",
    "\n",
    "            l2_loss = l2_lambda * tf.losses.get_regularization_loss()\n",
    "\n",
    "            total_loss = tf.cond(pretrain, lambda:loss_pos, lambda:loss + loss_pos + l2_loss)\n",
    "\n",
    "            summaries.append(tf.summary.scalar(\"loss_{}\".format(gpu_idx), loss))\n",
    "            summaries.append(tf.summary.scalar(\"loss_pos_{}\".format(gpu_idx), loss_pos))\n",
    "            summaries.append(tf.summary.scalar(\"total_loss_{}\".format(gpu_idx), total_loss))\n",
    "\n",
    "\n",
    "            optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "            grads_vars = optimizer.compute_gradients(total_loss)\n",
    "\n",
    "            clipped_grads = grads_vars\n",
    "            if(clipping == True):\n",
    "                clipped_grads = [(tf.clip_by_norm(grad, clip_norm), var) for grad, var in clipped_grads]\n",
    "\n",
    "            tower_grads.append(clipped_grads)\n",
    "            losses.append(total_loss)\n",
    "\n",
    "with tf.device('/gpu:0'):\n",
    "    tower_grads = average_gradients(tower_grads)\n",
    "    losses = tf.add_n(losses)/len(losses)\n",
    "    apply_grad_op = optimizer.apply_gradients(tower_grads, global_step=global_step)\n",
    "    summaries.append(tf.summary.scalar('total_loss', losses))\n",
    "    summaries.append(tf.summary.scalar('learning_rate', learning_rate))\n",
    "\n",
    "    variable_averages = tf.train.ExponentialMovingAverage(moving_avg_deacy, global_step)\n",
    "    variables_averages_op = variable_averages.apply(tf.trainable_variables())\n",
    "\n",
    "    train_op = tf.group(apply_grad_op, variables_averages_op)\n",
    "    saver = tf.train.Saver(tf.global_variables())\n",
    "    summary = tf.summary.merge(summaries)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.environ[\"CUDA_DEVICE_ORDER\"]=\"PCI_BUS_ID\"   # see issue #152\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0,1\"\n",
    "# print (device_lib.list_local_devices())\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth = True\n",
    "config.allow_soft_placement = True\n",
    "sess = tf.Session(config=config)\n",
    "sess.run(tf.global_variables_initializer())                          # For initializing all the variables\n",
    "summary_writer = tf.summary.FileWriter(log_dir, sess.graph)          # For writing Summaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_period = 100\n",
    "log_period = 100\n",
    "\n",
    "def model(xx, yy, yy_pos, mask, smask, train_cond=True, pretrain_cond=False):\n",
    "    num_batches = int(len(xx)/(batch_size*num_gpus))\n",
    "    _losses = 0\n",
    "    temp_loss = 0\n",
    "    preds_sense = []\n",
    "    true_sense = []\n",
    "    preds_pos = []\n",
    "    true_pos = []\n",
    "    \n",
    "    for j in range(num_batches): \n",
    "        \n",
    "        s = j * batch_size * num_gpus\n",
    "        e = (j+1) * batch_size * num_gpus\n",
    "        xx_re = xx[s:e].reshape([num_gpus, batch_size, -1])\n",
    "        yy_re = yy[s:e].reshape([num_gpus, batch_size, -1])\n",
    "        yy_pos_re = yy_pos[s:e].reshape([num_gpus, batch_size, -1])\n",
    "        mask_re = mask[s:e].reshape([num_gpus, batch_size, -1])\n",
    "        smask_re = smask[s:e].reshape([num_gpus, batch_size, -1])\n",
    " \n",
    "        feed_dict = {x:xx_re, y:yy_re, y_pos:yy_pos_re, x_mask:mask_re, sense_mask:smask_re, pretrain:pretrain_cond, is_train:train_cond, input_keep_prob:keep_prob, word_emb_mat:word_embedding}\n",
    "        \n",
    "        if(train_cond==True):\n",
    "            _, _loss, step, _summary = sess.run([train_op, losses, global_step, summary], feed_dict)\n",
    "            summary_writer.add_summary(_summary, step)\n",
    "            \n",
    "            temp_loss += _loss\n",
    "            if((j+1)%log_period==0):\n",
    "                print(\"Steps: {}\".format(step), \"Loss:{0:.4f}\".format(temp_loss/log_period), \", Current Loss: {0:.4f}\".format(_loss))\n",
    "                temp_loss = 0\n",
    "            if((j+1)%save_period==0):\n",
    "                saver.save(sess, save_path=save_dir)                         \n",
    "                \n",
    "        else:\n",
    "            _loss, pred, pred_pos = sess.run([total_loss, predictions, predictions_pos], feed_dict)\n",
    "            for i in range(num_gpus):\n",
    "                preds_sense.append(pred[i][smask_re[i]])\n",
    "                true_sense.append(yy_re[i][smask_re[i]])\n",
    "                preds_pos.append(pred_pos[i][mask_re[i]])\n",
    "                true_pos.append(yy_pos_re[i][mask_re[i]])\n",
    "\n",
    "        _losses +=_loss\n",
    "\n",
    "    if(train_cond==False): \n",
    "        sense_preds = []\n",
    "        sense_true = []\n",
    "        pos_preds = []\n",
    "        pos_true = []\n",
    "        \n",
    "        for preds in preds_sense:\n",
    "            for ps in preds:      \n",
    "                sense_preds.append(ps)  \n",
    "        for trues in true_sense:\n",
    "            for ts in trues:\n",
    "                sense_true.append(ts)\n",
    "        \n",
    "        for preds in preds_pos:\n",
    "            for ps in preds:      \n",
    "                pos_preds.append(ps)      \n",
    "        for trues in true_pos:\n",
    "            for ts in trues:\n",
    "                pos_true.append(ts)\n",
    "                \n",
    "        return _losses/num_batches, sense_preds, sense_true, pos_preds, pos_true\n",
    "\n",
    "    return _losses/num_batches, step\n",
    "\n",
    "def eval_score(yy, pred, yy_pos, pred_pos):\n",
    "    f1 = f1_score(yy, pred, average='macro')\n",
    "    accu = accuracy_score(yy, pred)\n",
    "    f1_pos = f1_score(yy_pos, pred_pos, average='macro')\n",
    "    accu_pos = accuracy_score(yy_pos, pred_pos)\n",
    "    return f1*100, accu*100, f1_pos*100, accu_pos*100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_id_train = train_data['x']\n",
    "mask_train = train_data['x_mask']\n",
    "sense_mask_train = train_data['sense_mask']\n",
    "y_train = train_data['y']\n",
    "y_pos_train = train_data['pos']\n",
    "\n",
    "x_id_val = val_data['x']\n",
    "mask_val = val_data['x_mask']\n",
    "sense_mask_val = val_data['sense_mask']\n",
    "y_val = val_data['y']\n",
    "y_pos_val = val_data['pos']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def testing():\n",
    "    start_time = time.time()\n",
    "    val_loss, val_pred, val_true, val_pred_pos, val_true_pos = model(x_id_val, y_val, y_pos_val, mask_val, sense_mask_val, train_cond=False)        \n",
    "    f1_, accu_, f1_pos_, accu_pos_ = eval_score(val_true, val_pred, val_true_pos, val_pred_pos)\n",
    "    time_taken = time.time() - start_time\n",
    "    print(\"Val: F1 Score:{0:.2f}\".format(f1_), \"Accuracy:{0:.2f}\".format(accu_), \" POS: F1 Score:{0:.2f}\".format(f1_pos_), \"Accuracy:{0:.2f}\".format(accu_pos_), \"Loss:{0:.4f}\".format(val_loss), \", Time: {0:.1f}\".format(time_taken))\n",
    "    return f1_, accu_, f1_pos_, accu_pos_\n",
    "\n",
    "def training(current_epoch, pre_train_cond):\n",
    "        random = np.random.choice(len(y_train), size=(len(y_train)), replace=False)\n",
    "        x_id_train_tmp = x_id_train[random]\n",
    "        y_train_tmp = y_train[random]\n",
    "        mask_train_tmp = mask_train[random]    \n",
    "        sense_mask_train_tmp = sense_mask_train[random]\n",
    "        y_pos_train_tmp = y_pos_train[random]\n",
    "\n",
    "        start_time = time.time()\n",
    "        train_loss, step = model(x_id_train_tmp, y_train_tmp, y_pos_train_tmp, mask_train_tmp, sense_mask_train_tmp, pretrain_cond=pre_train_cond)\n",
    "        time_taken = time.time() - start_time\n",
    "        print(\"Epoch: {}\".format(current_epoch+1),\", Step: {}\".format(step), \", loss: {0:.4f}\".format(train_loss), \", Time: {0:.1f}\".format(time_taken))\n",
    "        saver.save(sess, save_path=save_dir)                         \n",
    "        print(\"Model Saved\")\n",
    "        return [step, train_loss]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Steps: 100 Loss:1.9982 , Current Loss: 1.8051\n",
      "Steps: 200 Loss:1.7197 , Current Loss: 1.6263\n",
      "Steps: 300 Loss:1.4362 , Current Loss: 1.2040\n",
      "Steps: 400 Loss:0.9912 , Current Loss: 0.8256\n",
      "Steps: 500 Loss:0.7484 , Current Loss: 0.6934\n",
      "Steps: 600 Loss:0.6320 , Current Loss: 0.5834\n",
      "Steps: 700 Loss:0.5568 , Current Loss: 0.5258\n",
      "Steps: 800 Loss:0.5060 , Current Loss: 0.4828\n",
      "Epoch: 1 , Step: 885 , loss: 1.0153 , Time: 5979.3\n",
      "Model Saved\n",
      "Steps: 985 Loss:0.4392 , Current Loss: 0.4266\n",
      "Steps: 1085 Loss:0.4139 , Current Loss: 0.3861\n",
      "Steps: 1185 Loss:0.3937 , Current Loss: 0.3861\n",
      "Steps: 1285 Loss:0.3774 , Current Loss: 0.3727\n",
      "Steps: 1385 Loss:0.3636 , Current Loss: 0.3633\n",
      "Steps: 1485 Loss:0.3507 , Current Loss: 0.3340\n",
      "Steps: 1585 Loss:0.3399 , Current Loss: 0.3269\n",
      "Steps: 1685 Loss:0.3306 , Current Loss: 0.3224\n",
      "Epoch: 2 , Step: 1770 , loss: 0.3710 , Time: 5582.7\n",
      "Model Saved\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/users/btech/aviraj/envs/lib/python3.5/site-packages/sklearn/metrics/classification.py:1137: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val: F1 Score:1.32 Accuracy:1.85  POS: F1 Score:81.40 Accuracy:89.15 Loss:7.4181 , Time: 1059.9\n",
      "Steps: 1870 Loss:3.3076 , Current Loss: 3.0092\n",
      "Steps: 1970 Loss:2.8784 , Current Loss: 2.8019\n",
      "Steps: 2070 Loss:2.7729 , Current Loss: 2.7046\n",
      "Steps: 2170 Loss:2.7062 , Current Loss: 2.6756\n",
      "Steps: 2270 Loss:2.6582 , Current Loss: 2.6097\n",
      "Steps: 2370 Loss:2.6088 , Current Loss: 2.5857\n",
      "Steps: 2470 Loss:2.5835 , Current Loss: 2.6127\n",
      "Steps: 2570 Loss:2.5519 , Current Loss: 2.5605\n",
      "Epoch: 1 , Step: 2655 , loss: 2.7366 , Time: 5577.1\n",
      "Model Saved\n",
      "Steps: 2755 Loss:2.4972 , Current Loss: 2.5024\n",
      "Steps: 2855 Loss:2.4838 , Current Loss: 2.4786\n",
      "Steps: 2955 Loss:2.4659 , Current Loss: 2.4545\n",
      "Steps: 3055 Loss:2.4555 , Current Loss: 2.5147\n",
      "Steps: 3155 Loss:2.4404 , Current Loss: 2.3822\n",
      "Steps: 3255 Loss:2.4216 , Current Loss: 2.3864\n",
      "Steps: 3355 Loss:2.4090 , Current Loss: 2.3548\n",
      "Steps: 3455 Loss:2.3909 , Current Loss: 2.4483\n",
      "Epoch: 2 , Step: 3540 , loss: 2.4393 , Time: 5560.8\n",
      "Model Saved\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/users/btech/aviraj/envs/lib/python3.5/site-packages/sklearn/metrics/classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val: F1 Score:23.64 Accuracy:37.88  POS: F1 Score:78.70 Accuracy:87.95 Loss:2.3828 , Time: 1056.9\n",
      "Steps: 3640 Loss:2.3648 , Current Loss: 2.3518\n",
      "Steps: 3740 Loss:2.3621 , Current Loss: 2.3801\n",
      "Steps: 3840 Loss:2.3520 , Current Loss: 2.3743\n",
      "Steps: 3940 Loss:2.3326 , Current Loss: 2.4587\n",
      "Steps: 4040 Loss:2.3374 , Current Loss: 2.3275\n",
      "Steps: 4140 Loss:2.3195 , Current Loss: 2.3063\n",
      "Steps: 4240 Loss:2.3023 , Current Loss: 2.3698\n",
      "Steps: 4340 Loss:2.3039 , Current Loss: 2.2764\n",
      "Epoch: 3 , Step: 4425 , loss: 2.3296 , Time: 5570.7\n",
      "Model Saved\n",
      "Steps: 4525 Loss:2.2810 , Current Loss: 2.2548\n",
      "Steps: 4625 Loss:2.2685 , Current Loss: 2.2344\n",
      "Steps: 4725 Loss:2.2680 , Current Loss: 2.2505\n",
      "Steps: 4825 Loss:2.2593 , Current Loss: 2.1693\n",
      "Steps: 4925 Loss:2.2512 , Current Loss: 2.2359\n",
      "Steps: 5025 Loss:2.2460 , Current Loss: 2.2229\n",
      "Steps: 5125 Loss:2.2348 , Current Loss: 2.2192\n",
      "Steps: 5225 Loss:2.2338 , Current Loss: 2.3266\n",
      "Epoch: 4 , Step: 5310 , loss: 2.2530 , Time: 5585.4\n",
      "Model Saved\n",
      "Val: F1 Score:28.57 Accuracy:41.90  POS: F1 Score:79.47 Accuracy:88.60 Loss:2.2314 , Time: 1063.0\n",
      "Steps: 5410 Loss:2.2172 , Current Loss: 2.2953\n",
      "Steps: 5510 Loss:2.2113 , Current Loss: 2.1337\n",
      "Steps: 5610 Loss:2.2093 , Current Loss: 2.1773\n",
      "Steps: 5710 Loss:2.2040 , Current Loss: 2.2831\n",
      "Steps: 5810 Loss:2.1975 , Current Loss: 2.1972\n",
      "Steps: 5910 Loss:2.1908 , Current Loss: 2.1304\n",
      "Steps: 6010 Loss:2.1832 , Current Loss: 2.1333\n",
      "Steps: 6110 Loss:2.1829 , Current Loss: 2.1991\n",
      "Epoch: 5 , Step: 6195 , loss: 2.1971 , Time: 6184.9\n",
      "Model Saved\n",
      "Steps: 6295 Loss:2.1690 , Current Loss: 2.1818\n",
      "Steps: 6395 Loss:2.1549 , Current Loss: 2.2061\n",
      "Steps: 6495 Loss:2.1593 , Current Loss: 2.0972\n",
      "Steps: 6595 Loss:2.1617 , Current Loss: 2.2053\n",
      "Steps: 6695 Loss:2.1501 , Current Loss: 2.1399\n",
      "Steps: 6795 Loss:2.1530 , Current Loss: 2.1931\n",
      "Steps: 6895 Loss:2.1362 , Current Loss: 2.0376\n",
      "Steps: 6995 Loss:2.1376 , Current Loss: 2.1681\n",
      "Epoch: 6 , Step: 7080 , loss: 2.1508 , Time: 6891.8\n",
      "Model Saved\n",
      "Val: F1 Score:31.67 Accuracy:44.05  POS: F1 Score:79.77 Accuracy:88.93 Loss:2.1431 , Time: 1358.0\n",
      "Steps: 7180 Loss:2.1281 , Current Loss: 2.1690\n",
      "Steps: 7280 Loss:2.1202 , Current Loss: 2.1013\n",
      "Steps: 7380 Loss:2.1275 , Current Loss: 2.1216\n",
      "Steps: 7480 Loss:2.1099 , Current Loss: 2.0833\n",
      "Steps: 7580 Loss:2.1050 , Current Loss: 2.1486\n",
      "Steps: 7680 Loss:2.1155 , Current Loss: 2.1643\n",
      "Steps: 7780 Loss:2.1156 , Current Loss: 2.1748\n",
      "Steps: 7880 Loss:2.0993 , Current Loss: 2.0887\n",
      "Epoch: 7 , Step: 7965 , loss: 2.1125 , Time: 6158.7\n",
      "Model Saved\n",
      "Steps: 8065 Loss:2.0885 , Current Loss: 2.1221\n",
      "Steps: 8165 Loss:2.0913 , Current Loss: 2.0929\n",
      "Steps: 8265 Loss:2.0740 , Current Loss: 2.0358\n",
      "Steps: 8365 Loss:2.0793 , Current Loss: 2.0464\n",
      "Steps: 8465 Loss:2.0769 , Current Loss: 2.1512\n",
      "Steps: 8565 Loss:2.0805 , Current Loss: 2.0170\n",
      "Steps: 8665 Loss:2.0717 , Current Loss: 2.0515\n",
      "Steps: 8765 Loss:2.0798 , Current Loss: 2.1126\n",
      "Epoch: 8 , Step: 8850 , loss: 2.0792 , Time: 5814.6\n",
      "Model Saved\n",
      "Val: F1 Score:33.64 Accuracy:45.74  POS: F1 Score:80.39 Accuracy:89.30 Loss:2.0707 , Time: 1073.3\n",
      "Steps: 8950 Loss:2.0669 , Current Loss: 2.1054\n",
      "Steps: 9050 Loss:2.0555 , Current Loss: 2.0795\n",
      "Steps: 9150 Loss:2.0607 , Current Loss: 2.0450\n",
      "Steps: 9250 Loss:2.0480 , Current Loss: 2.0569\n",
      "Steps: 9350 Loss:2.0546 , Current Loss: 2.0378\n",
      "Steps: 9450 Loss:2.0490 , Current Loss: 1.9500\n",
      "Steps: 9550 Loss:2.0436 , Current Loss: 2.0423\n",
      "Steps: 9650 Loss:2.0399 , Current Loss: 2.0563\n",
      "Epoch: 9 , Step: 9735 , loss: 2.0509 , Time: 5655.1\n",
      "Model Saved\n",
      "Steps: 9835 Loss:2.0256 , Current Loss: 2.0590\n",
      "Steps: 9935 Loss:2.0235 , Current Loss: 2.0265\n",
      "Steps: 10035 Loss:2.0395 , Current Loss: 2.0859\n",
      "Steps: 10135 Loss:2.0212 , Current Loss: 2.0753\n",
      "Steps: 10235 Loss:2.0330 , Current Loss: 2.1014\n",
      "Steps: 10335 Loss:2.0245 , Current Loss: 2.0179\n",
      "Steps: 10435 Loss:2.0232 , Current Loss: 2.0612\n",
      "Steps: 10535 Loss:2.0140 , Current Loss: 1.9865\n",
      "Epoch: 10 , Step: 10620 , loss: 2.0240 , Time: 5858.9\n",
      "Model Saved\n",
      "Val: F1 Score:35.60 Accuracy:47.24  POS: F1 Score:80.65 Accuracy:89.48 Loss:2.0260 , Time: 1162.0\n",
      "Steps: 10720 Loss:2.0157 , Current Loss: 1.9840\n",
      "Steps: 10820 Loss:2.0101 , Current Loss: 1.9814\n",
      "Steps: 10920 Loss:2.0002 , Current Loss: 2.0668\n",
      "Steps: 11020 Loss:1.9923 , Current Loss: 1.9457\n",
      "Steps: 11120 Loss:2.0025 , Current Loss: 1.9797\n",
      "Steps: 11220 Loss:2.0066 , Current Loss: 1.9639\n",
      "Steps: 11320 Loss:2.0012 , Current Loss: 2.0077\n",
      "Steps: 11420 Loss:1.9938 , Current Loss: 1.9399\n",
      "Epoch: 11 , Step: 11505 , loss: 2.0019 , Time: 6362.6\n",
      "Model Saved\n",
      "Steps: 11605 Loss:1.9858 , Current Loss: 1.9818\n",
      "Steps: 11705 Loss:1.9858 , Current Loss: 2.0095\n",
      "Steps: 11805 Loss:1.9793 , Current Loss: 2.0627\n",
      "Steps: 11905 Loss:1.9905 , Current Loss: 2.0146\n",
      "Steps: 12005 Loss:1.9731 , Current Loss: 1.9373\n",
      "Steps: 12105 Loss:1.9758 , Current Loss: 1.9756\n",
      "Steps: 12205 Loss:1.9789 , Current Loss: 2.0288\n",
      "Steps: 12305 Loss:1.9715 , Current Loss: 1.9417\n",
      "Epoch: 12 , Step: 12390 , loss: 1.9801 , Time: 6831.6\n",
      "Model Saved\n",
      "Val: F1 Score:36.14 Accuracy:48.20  POS: F1 Score:80.93 Accuracy:89.68 Loss:1.9884 , Time: 1215.2\n",
      "Steps: 12490 Loss:1.9669 , Current Loss: 1.9400\n",
      "Steps: 12590 Loss:1.9671 , Current Loss: 1.9563\n",
      "Steps: 12690 Loss:1.9746 , Current Loss: 1.9573\n",
      "Steps: 12790 Loss:1.9627 , Current Loss: 1.8973\n",
      "Steps: 12890 Loss:1.9587 , Current Loss: 1.9188\n",
      "Steps: 12990 Loss:1.9568 , Current Loss: 1.9328\n",
      "Steps: 13090 Loss:1.9592 , Current Loss: 2.0382\n",
      "Steps: 13190 Loss:1.9580 , Current Loss: 1.9441\n",
      "Epoch: 13 , Step: 13275 , loss: 1.9615 , Time: 6040.2\n",
      "Model Saved\n",
      "Steps: 13375 Loss:1.9557 , Current Loss: 1.9084\n",
      "Steps: 13475 Loss:1.9430 , Current Loss: 1.8654\n",
      "Steps: 13575 Loss:1.9478 , Current Loss: 1.9992\n",
      "Steps: 13675 Loss:1.9460 , Current Loss: 1.9174\n",
      "Steps: 13775 Loss:1.9520 , Current Loss: 1.9184\n",
      "Steps: 13875 Loss:1.9424 , Current Loss: 1.9587\n",
      "Steps: 13975 Loss:1.9406 , Current Loss: 1.7941\n",
      "Steps: 14075 Loss:1.9398 , Current Loss: 1.9498\n",
      "Epoch: 14 , Step: 14160 , loss: 1.9450 , Time: 5589.7\n",
      "Model Saved\n",
      "Val: F1 Score:37.83 Accuracy:49.18  POS: F1 Score:81.11 Accuracy:89.81 Loss:1.9514 , Time: 1064.0\n",
      "Steps: 14260 Loss:1.9331 , Current Loss: 1.8827\n",
      "Steps: 14360 Loss:1.9248 , Current Loss: 1.9128\n",
      "Steps: 14460 Loss:1.9361 , Current Loss: 2.0662\n",
      "Steps: 14560 Loss:1.9307 , Current Loss: 1.8387\n",
      "Steps: 14660 Loss:1.9268 , Current Loss: 1.9213\n",
      "Steps: 14760 Loss:1.9256 , Current Loss: 1.9286\n",
      "Steps: 14860 Loss:1.9332 , Current Loss: 1.9399\n",
      "Steps: 14960 Loss:1.9278 , Current Loss: 1.9692\n",
      "Epoch: 15 , Step: 15045 , loss: 1.9277 , Time: 5576.7\n",
      "Model Saved\n",
      "Steps: 15145 Loss:1.9184 , Current Loss: 1.8768\n",
      "Steps: 15245 Loss:1.9201 , Current Loss: 1.9362\n",
      "Steps: 15345 Loss:1.9143 , Current Loss: 1.8484\n",
      "Steps: 15445 Loss:1.9189 , Current Loss: 1.8972\n",
      "Steps: 15545 Loss:1.9139 , Current Loss: 1.9082\n",
      "Steps: 15645 Loss:1.9052 , Current Loss: 1.9789\n",
      "Steps: 15745 Loss:1.9142 , Current Loss: 1.9022\n",
      "Steps: 15845 Loss:1.9059 , Current Loss: 1.9689\n",
      "Epoch: 16 , Step: 15930 , loss: 1.9133 , Time: 5562.2\n",
      "Model Saved\n",
      "Val: F1 Score:39.05 Accuracy:50.12  POS: F1 Score:81.37 Accuracy:89.93 Loss:1.9142 , Time: 1049.6\n",
      "Steps: 16030 Loss:1.8969 , Current Loss: 1.8265\n",
      "Steps: 16130 Loss:1.9012 , Current Loss: 1.9214\n",
      "Steps: 16230 Loss:1.8999 , Current Loss: 1.9530\n",
      "Steps: 16330 Loss:1.9008 , Current Loss: 1.9428\n",
      "Steps: 16430 Loss:1.8974 , Current Loss: 1.9629\n",
      "Steps: 16530 Loss:1.8879 , Current Loss: 1.8402\n",
      "Steps: 16630 Loss:1.8968 , Current Loss: 1.8206\n",
      "Steps: 16730 Loss:1.9005 , Current Loss: 1.9696\n",
      "Epoch: 17 , Step: 16815 , loss: 1.8984 , Time: 5579.5\n",
      "Model Saved\n",
      "Steps: 16915 Loss:1.8880 , Current Loss: 1.8924\n",
      "Steps: 17015 Loss:1.8892 , Current Loss: 1.9388\n",
      "Steps: 17115 Loss:1.8906 , Current Loss: 1.9635\n",
      "Steps: 17215 Loss:1.8937 , Current Loss: 1.8366\n",
      "Steps: 17315 Loss:1.8891 , Current Loss: 1.8817\n",
      "Steps: 17415 Loss:1.8815 , Current Loss: 1.7406\n",
      "Steps: 17515 Loss:1.8861 , Current Loss: 1.9362\n",
      "Steps: 17615 Loss:1.8828 , Current Loss: 1.8695\n",
      "Epoch: 18 , Step: 17700 , loss: 1.8872 , Time: 7272.2\n",
      "Model Saved\n",
      "Val: F1 Score:39.87 Accuracy:50.77  POS: F1 Score:81.50 Accuracy:90.01 Loss:1.8956 , Time: 1243.7\n",
      "Steps: 17800 Loss:1.8805 , Current Loss: 1.9463\n",
      "Steps: 17900 Loss:1.8796 , Current Loss: 1.8903\n",
      "Steps: 18000 Loss:1.8720 , Current Loss: 1.8301\n",
      "Steps: 18100 Loss:1.8717 , Current Loss: 1.9199\n",
      "Steps: 18200 Loss:1.8801 , Current Loss: 1.9648\n",
      "Steps: 18300 Loss:1.8698 , Current Loss: 1.8150\n",
      "Steps: 18400 Loss:1.8710 , Current Loss: 1.8986\n",
      "Steps: 18500 Loss:1.8676 , Current Loss: 1.8580\n",
      "Epoch: 19 , Step: 18585 , loss: 1.8745 , Time: 7665.2\n",
      "Model Saved\n",
      "Steps: 18685 Loss:1.8616 , Current Loss: 1.9140\n",
      "Steps: 18785 Loss:1.8549 , Current Loss: 1.8697\n",
      "Steps: 18885 Loss:1.8681 , Current Loss: 1.7750\n",
      "Steps: 18985 Loss:1.8622 , Current Loss: 1.7742\n",
      "Steps: 19085 Loss:1.8624 , Current Loss: 1.9213\n",
      "Steps: 19185 Loss:1.8589 , Current Loss: 1.8596\n",
      "Steps: 19285 Loss:1.8661 , Current Loss: 1.8454\n",
      "Steps: 19385 Loss:1.8703 , Current Loss: 1.8987\n",
      "Epoch: 20 , Step: 19470 , loss: 1.8636 , Time: 6894.3\n",
      "Model Saved\n",
      "Val: F1 Score:40.95 Accuracy:51.61  POS: F1 Score:81.65 Accuracy:90.15 Loss:1.8717 , Time: 1071.7\n"
     ]
    }
   ],
   "source": [
    "loss_collection = []\n",
    "val_collection = []\n",
    "num_epochs = 20\n",
    "val_period = 2\n",
    "\n",
    "# Pretraining POS Tags\n",
    "training(0, True)\n",
    "training(1, True)\n",
    "testing()\n",
    "\n",
    "for i in range(num_epochs):\n",
    "    loss_collection.append(training(i, False))\n",
    "    if((i+1)%val_period==0):\n",
    "        val_collection.append(testing())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Steps: 19570 Loss:1.8589 , Current Loss: 1.8295\n",
      "Steps: 19670 Loss:1.8433 , Current Loss: 1.7991\n",
      "Steps: 19770 Loss:1.8585 , Current Loss: 1.8554\n",
      "Steps: 19870 Loss:1.8477 , Current Loss: 1.7974\n",
      "Steps: 19970 Loss:1.8521 , Current Loss: 1.8273\n",
      "Steps: 20070 Loss:1.8499 , Current Loss: 1.8567\n",
      "Steps: 20170 Loss:1.8474 , Current Loss: 1.8464\n",
      "Steps: 20270 Loss:1.8503 , Current Loss: 1.8766\n",
      "Epoch: 1 , Step: 20355 , loss: 1.8508 , Time: 7720.8\n",
      "Model Saved\n",
      "Steps: 20455 Loss:1.8415 , Current Loss: 1.8565\n",
      "Steps: 20555 Loss:1.8419 , Current Loss: 1.8260\n",
      "Steps: 20655 Loss:1.8429 , Current Loss: 1.7780\n",
      "Steps: 20755 Loss:1.8468 , Current Loss: 1.8108\n",
      "Steps: 20855 Loss:1.8380 , Current Loss: 1.8209\n",
      "Steps: 20955 Loss:1.8350 , Current Loss: 1.7977\n",
      "Steps: 21055 Loss:1.8404 , Current Loss: 1.8154\n",
      "Steps: 21155 Loss:1.8427 , Current Loss: 1.7872\n",
      "Epoch: 2 , Step: 21240 , loss: 1.8410 , Time: 9505.3\n",
      "Model Saved\n",
      "Val: F1 Score:41.04 Accuracy:52.05  POS: F1 Score:81.68 Accuracy:90.19 Loss:1.8493 , Time: 2170.0\n",
      "Steps: 21340 Loss:1.8344 , Current Loss: 1.8264\n",
      "Steps: 21440 Loss:1.8300 , Current Loss: 1.9880\n",
      "Steps: 21540 Loss:1.8364 , Current Loss: 1.8929\n",
      "Steps: 21640 Loss:1.8344 , Current Loss: 1.7797\n",
      "Steps: 21740 Loss:1.8317 , Current Loss: 1.8357\n",
      "Steps: 21840 Loss:1.8369 , Current Loss: 1.8498\n",
      "Steps: 21940 Loss:1.8217 , Current Loss: 1.8316\n",
      "Steps: 22040 Loss:1.8315 , Current Loss: 1.8486\n",
      "Epoch: 3 , Step: 22125 , loss: 1.8310 , Time: 9919.3\n",
      "Model Saved\n",
      "Steps: 22225 Loss:1.8191 , Current Loss: 1.7856\n",
      "Steps: 22325 Loss:1.8250 , Current Loss: 1.8690\n",
      "Steps: 22425 Loss:1.8256 , Current Loss: 1.8279\n",
      "Steps: 22525 Loss:1.8226 , Current Loss: 1.8819\n",
      "Steps: 22625 Loss:1.8199 , Current Loss: 1.8674\n",
      "Steps: 22725 Loss:1.8189 , Current Loss: 1.8331\n",
      "Steps: 22825 Loss:1.8223 , Current Loss: 1.8848\n",
      "Steps: 22925 Loss:1.8272 , Current Loss: 1.7457\n",
      "Epoch: 4 , Step: 23010 , loss: 1.8228 , Time: 9329.0\n",
      "Model Saved\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/users/btech/aviraj/envs/lib/python3.5/site-packages/sklearn/metrics/classification.py:1137: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val: F1 Score:41.02 Accuracy:52.48  POS: F1 Score:81.99 Accuracy:90.28 Loss:1.8354 , Time: 1840.1\n",
      "Steps: 23110 Loss:1.8162 , Current Loss: 1.7691\n",
      "Steps: 23210 Loss:1.8143 , Current Loss: 1.8730\n",
      "Steps: 23310 Loss:1.8107 , Current Loss: 1.8690\n",
      "Steps: 23410 Loss:1.8112 , Current Loss: 1.7655\n",
      "Steps: 23510 Loss:1.8188 , Current Loss: 1.8089\n",
      "Steps: 23610 Loss:1.8141 , Current Loss: 1.7024\n",
      "Steps: 23710 Loss:1.7992 , Current Loss: 1.7244\n",
      "Steps: 23810 Loss:1.8144 , Current Loss: 1.7922\n",
      "Epoch: 5 , Step: 23895 , loss: 1.8123 , Time: 9325.3\n",
      "Model Saved\n",
      "Steps: 23995 Loss:1.8133 , Current Loss: 1.8444\n",
      "Steps: 24095 Loss:1.7980 , Current Loss: 1.8262\n",
      "Steps: 24195 Loss:1.8108 , Current Loss: 1.8121\n",
      "Steps: 24295 Loss:1.8058 , Current Loss: 1.8164\n",
      "Steps: 24395 Loss:1.7959 , Current Loss: 1.7693\n",
      "Steps: 24495 Loss:1.8047 , Current Loss: 1.8404\n",
      "Steps: 24595 Loss:1.8028 , Current Loss: 1.8024\n",
      "Steps: 24695 Loss:1.8065 , Current Loss: 1.9289\n",
      "Epoch: 6 , Step: 24780 , loss: 1.8058 , Time: 9360.8\n",
      "Model Saved\n",
      "Val: F1 Score:42.96 Accuracy:52.95  POS: F1 Score:82.06 Accuracy:90.36 Loss:1.8219 , Time: 2011.5\n",
      "Steps: 24880 Loss:1.7999 , Current Loss: 1.8021\n",
      "Steps: 24980 Loss:1.7984 , Current Loss: 1.8373\n",
      "Steps: 25080 Loss:1.7986 , Current Loss: 1.7742\n",
      "Steps: 25180 Loss:1.8109 , Current Loss: 1.7967\n",
      "Steps: 25280 Loss:1.8013 , Current Loss: 1.7873\n",
      "Steps: 25380 Loss:1.7897 , Current Loss: 1.8493\n",
      "Steps: 25480 Loss:1.7925 , Current Loss: 1.8990\n",
      "Steps: 25580 Loss:1.7886 , Current Loss: 1.7829\n",
      "Epoch: 7 , Step: 25665 , loss: 1.7967 , Time: 10323.8\n",
      "Model Saved\n",
      "Steps: 25765 Loss:1.7924 , Current Loss: 1.8184\n",
      "Steps: 25865 Loss:1.7870 , Current Loss: 1.6967\n",
      "Steps: 25965 Loss:1.7938 , Current Loss: 1.8052\n",
      "Steps: 26065 Loss:1.7943 , Current Loss: 1.7534\n",
      "Steps: 26165 Loss:1.7830 , Current Loss: 1.8248\n",
      "Steps: 26265 Loss:1.7993 , Current Loss: 1.8132\n",
      "Steps: 26365 Loss:1.7895 , Current Loss: 1.8668\n",
      "Steps: 26465 Loss:1.7800 , Current Loss: 1.8191\n",
      "Epoch: 8 , Step: 26550 , loss: 1.7884 , Time: 10274.2\n",
      "Model Saved\n"
     ]
    }
   ],
   "source": [
    "loss_collection = []\n",
    "val_collection = []\n",
    "num_epochs = 20\n",
    "val_period = 2\n",
    "\n",
    "for i in range(num_epochs):\n",
    "    loss_collection.append(training(i, False))\n",
    "    if((i+1)%val_period==0):\n",
    "        val_collection.append(testing())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Steps: 27735 Loss:1.7633 , Current Loss: 1.7947\n",
      "Steps: 27835 Loss:1.7800 , Current Loss: 1.7281\n",
      "Steps: 27935 Loss:1.7711 , Current Loss: 1.7880\n",
      "Steps: 28035 Loss:1.7737 , Current Loss: 1.6944\n",
      "Steps: 28135 Loss:1.7707 , Current Loss: 1.8615\n",
      "Steps: 28235 Loss:1.7716 , Current Loss: 1.8083\n",
      "Steps: 28335 Loss:1.7778 , Current Loss: 1.7110\n",
      "Steps: 28435 Loss:1.7693 , Current Loss: 1.7300\n",
      "Epoch: 1 , Step: 28520 , loss: 1.7721 , Time: 9042.6\n",
      "Model Saved\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0mTraceback (most recent call last)",
      "\u001b[0;32m<ipython-input-10-75cd3b5f5d64>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnum_epochs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m     \u001b[0mloss_collection\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtraining\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m     \u001b[0;32mif\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m%\u001b[0m\u001b[0mval_period\u001b[0m\u001b[0;34m==\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m         \u001b[0mval_collection\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtesting\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-8-781f9ea1ff25>\u001b[0m in \u001b[0;36mtraining\u001b[0;34m(current_epoch, pre_train_cond)\u001b[0m\n\u001b[1;32m     16\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m         \u001b[0mstart_time\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m         \u001b[0mtrain_loss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstep\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_id_train_tmp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train_tmp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_pos_train_tmp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmask_train_tmp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msense_mask_train_tmp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpretrain_cond\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpre_train_cond\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     19\u001b[0m         \u001b[0mtime_taken\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mstart_time\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Epoch: {}\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcurrent_epoch\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\", Step: {}\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\", loss: {0:.4f}\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_loss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\", Time: {0:.1f}\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtime_taken\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-6-84bb7f86a63d>\u001b[0m in \u001b[0;36mmodel\u001b[0;34m(xx, yy, yy_pos, mask, smask, train_cond, pretrain_cond)\u001b[0m\n\u001b[1;32m     24\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m         \u001b[0;32mif\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_cond\u001b[0m\u001b[0;34m==\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 26\u001b[0;31m             \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_loss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstep\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_summary\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtrain_op\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlosses\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mglobal_step\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msummary\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     27\u001b[0m             \u001b[0msummary_writer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_summary\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_summary\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     28\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    887\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    888\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 889\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    890\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    891\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1087\u001b[0m             \u001b[0mfeed_handles\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0msubfeed_t\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msubfeed_val\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1088\u001b[0m           \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1089\u001b[0;31m             \u001b[0mnp_val\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msubfeed_val\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msubfeed_dtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1090\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1091\u001b[0m           if (not is_tensor_handle_feed and\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/numpy/core/numeric.py\u001b[0m in \u001b[0;36masarray\u001b[0;34m(a, dtype, order)\u001b[0m\n\u001b[1;32m    490\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    491\u001b[0m     \"\"\"\n\u001b[0;32m--> 492\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0morder\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0morder\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    493\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    494\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "loss_collection = []\n",
    "val_collection = []\n",
    "num_epochs = 20\n",
    "val_period = 2\n",
    "\n",
    "for i in range(num_epochs):\n",
    "    loss_collection.append(training(i, False))\n",
    "    if((i+1)%val_period==0):\n",
    "        val_collection.append(testing())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val: F1 Score:44.36 Accuracy:53.76  POS: F1 Score:82.20 Accuracy:90.42 Loss:1.7889 , Time: 1421.9\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(44.361822318916865, 53.75801083454307, 82.19997565386215, 90.42074423342494)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testing()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "start_time = time.time()\n",
    "train_loss, train_pred, train_true, train_pred_pos, train_true_pos = model(x_id_train, y_train, y_pos_train, mask_train, sense_mask_train, train_cond=False)        \n",
    "f1_, accu_, f1_pos_, accu_pos_ = etrain_score(train_true, train_pred, train_true_pos, train_pred_pos)\n",
    "time_taken = time.time() - start_time\n",
    "print(\"train: F1 Score:{0:.2f}\".format(f1_), \"Accuracy:{0:.2f}\".format(accu_), \" POS: F1 Score:{0:.2f}\".format(f1_pos_), \"Accuracy:{0:.2f}\".format(accu_pos_), \"Loss:{0:.4f}\".format(train_loss), \", Time: {0:.1f}\".format(time_taken))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "saver.restore(sess, save_dir)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "envs",
   "language": "python",
   "name": "cs771"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}