one_million/force/Force-Model-2-multigpu-1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "tf.logging.set_verbosity(tf.logging.WARN)\n",
    "import pickle\n",
    "import numpy as np\n",
    "import os\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import accuracy_score\n",
    "import os\n",
    "from tensorflow.python.client import device_lib\n",
    "from collections import Counter\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('14', 1150), ('07', 969), ('41', 543), ('04', 495), ('18', 492), ('36', 31), ('19', 20), ('38', 10), ('32', 7), ('35', 6)]\n",
      "3649 3723\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/users/btech/aviraj/cs771/lib/python3.5/site-packages/sklearn/model_selection/_split.py:2026: FutureWarning: From version 0.21, test_size will always complement train_size unless both are specified.\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "f = open('../../Glove/word_embedding_glove', 'rb')\n",
    "word_embedding = pickle.load(f)\n",
    "f.close()\n",
    "word_embedding = word_embedding[: len(word_embedding)-1]\n",
    "\n",
    "f = open('../../Glove/vocab_glove', 'rb')\n",
    "vocab = pickle.load(f)\n",
    "f.close()\n",
    "\n",
    "word2id = dict((w, i) for i,w in enumerate(vocab))\n",
    "id2word = dict((i, w) for i,w in enumerate(vocab))\n",
    "\n",
    "unknown_token = \"UNKNOWN_TOKEN\"\n",
    "\n",
    "f = open(\"../../../dataset/sense/dict_sense-keys\", 'rb')\n",
    "dict_sense_keys = pickle.load(f)\n",
    "f.close()\n",
    "\n",
    "f = open(\"../../../dataset/sense/dict_word-sense\", 'rb')\n",
    "dict_word_sense = pickle.load(f)\n",
    "f.close()\n",
    "\n",
    "# Model Description\n",
    "sense_word = 'force'\n",
    "model_name = 'model-2-multigpu-1'\n",
    "sense_word_dir = '../output/' + sense_word\n",
    "model_dir = sense_word_dir + '/' + model_name\n",
    "save_dir = os.path.join(model_dir, \"save/\")\n",
    "log_dir = os.path.join(model_dir, \"log\")\n",
    "\n",
    "if not os.path.exists(sense_word_dir):\n",
    "    os.mkdir(sense_word_dir)\n",
    "if not os.path.exists(model_dir):\n",
    "    os.mkdir(model_dir)\n",
    "if not os.path.exists(save_dir):\n",
    "    os.mkdir(save_dir)\n",
    "if not os.path.exists(log_dir):\n",
    "    os.mkdir(log_dir)\n",
    "\n",
    "f = open(\"../../../dataset/checkwords/\"+ sense_word + \"_data\", 'rb')\n",
    "data = pickle.load(f)\n",
    "f.close()\n",
    "\n",
    "pos_tags = []\n",
    "for i in range(len(data)):\n",
    "    for p in data[i][2]:\n",
    "        pos_tags.append(p)\n",
    "        \n",
    "pos_count = Counter(pos_tags)\n",
    "pos_count = pos_count.most_common()\n",
    "vocab_pos = [k for k,v in pos_count]\n",
    "\n",
    "data_y = []\n",
    "for i in range(len(data)):\n",
    "    data_y.append(dict_sense_keys[data[i][0]][2])\n",
    "\n",
    "sense_count = Counter(data_y)\n",
    "sense_count = sense_count.most_common()\n",
    "vocab_sense = [k for k,v in sense_count[:5]]\n",
    "print(sense_count)\n",
    "\n",
    "data_x = []\n",
    "data_label = []\n",
    "data_pos = []\n",
    "for i in range(len(data)):\n",
    "    if dict_sense_keys[data[i][0]][2] in vocab_sense:\n",
    "        data_x.append(data[i][1])\n",
    "        data_pos.append(data[i][2])\n",
    "        data_label.append(dict_sense_keys[data[i][0]][2])\n",
    "\n",
    "print(len(data_label), len(data_y))\n",
    "\n",
    "pos2id = dict((s, i) for i,s in enumerate(vocab_pos))\n",
    "id2pos = dict((i, s) for i,s in enumerate(vocab_pos))\n",
    "\n",
    "# vocab_sense = dict_word_sense[sense_word]\n",
    "\n",
    "sense2id = dict((s, i) for i,s in enumerate(vocab_sense))\n",
    "id2sense = dict((i, s) for i,s in enumerate(vocab))\n",
    "\n",
    "len(vocab_pos)\n",
    "\n",
    "# Parameters\n",
    "mode = 'train'\n",
    "num_senses = len(vocab_sense)\n",
    "num_pos = len(vocab_pos)\n",
    "batch_size = 32\n",
    "vocab_size = len(vocab)\n",
    "unk_vocab_size = 1\n",
    "word_emb_size = len(word_embedding[0])\n",
    "max_sent_size = 200\n",
    "hidden_size = 100\n",
    "keep_prob = 0.5\n",
    "l2_lambda = 0.001\n",
    "init_lr = 0.005\n",
    "decay_steps = 500\n",
    "decay_rate = 0.96\n",
    "clip_norm = 1\n",
    "clipping = True\n",
    "pos_lambda = 5\n",
    "moving_avg_deacy = 0.999\n",
    "num_gpus = 6\n",
    "\n",
    "index = []\n",
    "for i in range(len(data_x)):\n",
    "    index.append(i)\n",
    "\n",
    "index_train, index_val, label_train, label_val = train_test_split(index, data_label, train_size=0.8, shuffle=True, stratify=data_label, random_state=0)\n",
    "\n",
    "data_x = np.array(data_x)\n",
    "data_pos = np.array(data_pos)\n",
    "\n",
    "x_train = data_x[index_train]\n",
    "pos_train = data_pos[index_train]\n",
    "x_val = data_x[index_val]\n",
    "pos_val = data_pos[index_val]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def average_gradients(tower_grads):\n",
    "    average_grads = []\n",
    "    for grad_and_vars in zip(*tower_grads):\n",
    "        # Note that each grad_and_vars looks like the following:\n",
    "        #   ((grad0_gpu0, var0_gpu0), ... , (grad0_gpuN, var0_gpuN))\n",
    "        grads = []\n",
    "        for g, _ in grad_and_vars:\n",
    "            # Add 0 dimension to the gradients to represent the tower.\n",
    "            expanded_g = tf.expand_dims(g, 0)\n",
    "\n",
    "            # Append on a 'tower' dimension which we will average over below.\n",
    "            grads.append(expanded_g)\n",
    "\n",
    "        # Average over the 'tower' dimension.\n",
    "        grad = tf.concat(grads, 0)\n",
    "        grad = tf.reduce_mean(grad, 0)\n",
    "\n",
    "        # Keep in mind that the Variables are redundant because they are shared\n",
    "        # across towers. So .. we will just return the first tower's pointer to\n",
    "        # the Variable.\n",
    "        v = grad_and_vars[0][1]\n",
    "        grad_and_var = (grad, v)\n",
    "        average_grads.append(grad_and_var)\n",
    "    return average_grads"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# MODEL\n",
    "tower_grads = []\n",
    "losses = []\n",
    "predictions = []\n",
    "predictions_pos = []\n",
    "\n",
    "x = tf.placeholder('int32', [num_gpus, batch_size, max_sent_size], name=\"x\")\n",
    "y = tf.placeholder('int32', [num_gpus, batch_size], name=\"y\")\n",
    "y_pos = tf.placeholder('int32', [num_gpus, batch_size, max_sent_size], name=\"y_pos\")\n",
    "x_mask  = tf.placeholder('bool', [num_gpus, batch_size, max_sent_size], name='x_mask') \n",
    "is_train = tf.placeholder('bool', [], name='is_train')\n",
    "word_emb_mat = tf.placeholder('float', [None, word_emb_size], name='emb_mat')\n",
    "input_keep_prob = tf.cond(is_train,lambda:keep_prob, lambda:tf.constant(1.0))\n",
    "global_step = tf.Variable(0, trainable=False, name=\"global_step\")\n",
    "learning_rate = tf.train.exponential_decay(init_lr, global_step, decay_steps, decay_rate, staircase=True)\n",
    "\n",
    "with tf.variable_scope(\"word_embedding\"):\n",
    "    unk_word_emb_mat = tf.get_variable(\"word_emb_mat\", dtype='float', shape=[unk_vocab_size, word_emb_size], initializer=tf.contrib.layers.xavier_initializer(uniform=True, seed=0, dtype=tf.float32))\n",
    "    final_word_emb_mat = tf.concat([word_emb_mat, unk_word_emb_mat], 0)\n",
    "\n",
    "with tf.variable_scope(tf.get_variable_scope()):\n",
    "    for gpu_idx in range(num_gpus):\n",
    "        with tf.name_scope(\"model_{}\".format(gpu_idx)) as scope, tf.device('/gpu:%d' % i):\n",
    "\n",
    "            if gpu_idx > 0:\n",
    "                    tf.get_variable_scope().reuse_variables()\n",
    "\n",
    "            with tf.name_scope(\"word\"):\n",
    "                Wx = tf.nn.embedding_lookup(final_word_emb_mat, x[gpu_idx])  \n",
    "\n",
    "            x_len = tf.reduce_sum(tf.cast(x_mask[gpu_idx], 'int32'), 1)\n",
    "\n",
    "            with tf.variable_scope(\"lstm1\"):\n",
    "                cell_fw1 = tf.contrib.rnn.BasicLSTMCell(hidden_size,state_is_tuple=True)\n",
    "                cell_bw1 = tf.contrib.rnn.BasicLSTMCell(hidden_size,state_is_tuple=True)\n",
    "\n",
    "                d_cell_fw1 = tf.contrib.rnn.DropoutWrapper(cell_fw1, input_keep_prob=input_keep_prob)\n",
    "                d_cell_bw1 = tf.contrib.rnn.DropoutWrapper(cell_bw1, input_keep_prob=input_keep_prob)\n",
    "\n",
    "                (fw_h1, bw_h1), _ = tf.nn.bidirectional_dynamic_rnn(d_cell_fw1, d_cell_bw1, Wx, sequence_length=x_len, dtype='float', scope='lstm1')\n",
    "                h1 = tf.concat([fw_h1, bw_h1], 2)\n",
    "\n",
    "            with tf.variable_scope(\"lstm2\"):\n",
    "                cell_fw2 = tf.contrib.rnn.BasicLSTMCell(hidden_size,state_is_tuple=True)\n",
    "                cell_bw2 = tf.contrib.rnn.BasicLSTMCell(hidden_size,state_is_tuple=True)\n",
    "\n",
    "                d_cell_fw2 = tf.contrib.rnn.DropoutWrapper(cell_fw2, input_keep_prob=input_keep_prob)\n",
    "                d_cell_bw2 = tf.contrib.rnn.DropoutWrapper(cell_bw2, input_keep_prob=input_keep_prob)\n",
    "\n",
    "                (fw_h2, bw_h2), _ = tf.nn.bidirectional_dynamic_rnn(d_cell_fw2, d_cell_bw2, h1, sequence_length=x_len, dtype='float', scope='lstm2')\n",
    "                h = tf.concat([fw_h2, bw_h2], 2)\n",
    "\n",
    "            def attention(input_x, input_mask, W_att):\n",
    "                h_masked = tf.boolean_mask(input_x, input_mask)\n",
    "                h_tanh = tf.tanh(h_masked)\n",
    "                u = tf.matmul(h_tanh, W_att)\n",
    "                a = tf.nn.softmax(u)\n",
    "                c = tf.reduce_sum(tf.multiply(h_tanh, a), 0)  \n",
    "                return c\n",
    "\n",
    "            with tf.variable_scope(\"attention\"):\n",
    "                W_att = tf.get_variable(\"W_att\", shape=[2*hidden_size, 1], initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1, seed=0))\n",
    "                c = tf.expand_dims(attention(h[0], x_mask[gpu_idx][0], W_att), 0)\n",
    "                for i in range(1, batch_size):\n",
    "                    c = tf.concat([c, tf.expand_dims(attention(h[i], x_mask[gpu_idx][i], W_att), 0)], 0)\n",
    "            \n",
    "            with tf.variable_scope(\"softmax_layer\"):\n",
    "                W = tf.get_variable(\"W\", shape=[2*hidden_size, num_senses], initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1, seed=0))\n",
    "                b = tf.get_variable(\"b\", shape=[num_senses], initializer=tf.zeros_initializer())\n",
    "                drop_c = tf.nn.dropout(c, input_keep_prob)\n",
    "                logits = tf.matmul(drop_c, W) + b\n",
    "                predictions.append(tf.argmax(logits, 1))\n",
    "\n",
    "            with tf.variable_scope(\"softmax_layer_pos\"):\n",
    "                W = tf.get_variable(\"W\", shape=[2*hidden_size, num_pos], initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1, seed=0))\n",
    "                b = tf.get_variable(\"b\", shape=[num_pos], initializer=tf.zeros_initializer())\n",
    "                flat_h1 = tf.reshape(h1, [-1, 2*hidden_size])\n",
    "                drop_flat_h1 = tf.nn.dropout(flat_h1, input_keep_prob)\n",
    "                flat_logits_pos = tf.matmul(drop_flat_h1, W) + b\n",
    "                logits_pos = tf.reshape(flat_logits_pos, [batch_size, max_sent_size, num_pos])\n",
    "                predictions_pos.append(tf.arg_max(logits_pos, 2))\n",
    "\n",
    "\n",
    "            float_x_mask = tf.cast(x_mask[gpu_idx], 'float')\n",
    "\n",
    "            loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=y[gpu_idx]))\n",
    "            loss_pos = pos_lambda*tf.contrib.seq2seq.sequence_loss(logits_pos, y_pos[gpu_idx], float_x_mask, name=\"loss_pos\")\n",
    "\n",
    "            l2_loss = l2_lambda * tf.losses.get_regularization_loss()\n",
    "\n",
    "            total_loss = loss + l2_loss + loss_pos\n",
    "            tf.summary.scalar(\"loss_{}\".format(gpu_idx), total_loss)\n",
    "\n",
    "            summaries = tf.get_collection(tf.GraphKeys.SUMMARIES, scope)\n",
    "\n",
    "            optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "            grads_vars = optimizer.compute_gradients(total_loss)\n",
    "\n",
    "            clipped_grads = grads_vars\n",
    "            if(clipping == True):\n",
    "                clipped_grads = [(tf.clip_by_norm(grad, clip_norm), var) for grad, var in clipped_grads]\n",
    "\n",
    "            tower_grads.append(clipped_grads)\n",
    "            losses.append(total_loss)\n",
    "\n",
    "tower_grads = average_gradients(tower_grads)\n",
    "losses = tf.add_n(losses)/len(losses)\n",
    "apply_grad_op = optimizer.apply_gradients(tower_grads, global_step=global_step)\n",
    "summaries.append(tf.summary.scalar('total_loss', losses))\n",
    "summaries.append(tf.summary.scalar('learning_rate', learning_rate))\n",
    "\n",
    "for var in tf.trainable_variables():\n",
    "    summaries.append(tf.summary.histogram(var.op.name, var))\n",
    "\n",
    "variable_averages = tf.train.ExponentialMovingAverage(moving_avg_deacy, global_step)\n",
    "variables_averages_op = variable_averages.apply(tf.trainable_variables())\n",
    "\n",
    "train_op = tf.group(apply_grad_op, variables_averages_op)\n",
    "saver = tf.train.Saver(tf.global_variables())\n",
    "summary = tf.summary.merge(summaries)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "os.environ[\"CUDA_DEVICE_ORDER\"]=\"PCI_BUS_ID\"   # see issue #152\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n",
    "# print (device_lib.list_local_devices())\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth = True\n",
    "config.allow_soft_placement = True\n",
    "sess = tf.Session(config=config)\n",
    "sess.run(tf.global_variables_initializer())                          # For initializing all the variables\n",
    "summary_writer = tf.summary.FileWriter(log_dir, sess.graph)          # For writing Summaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def data_prepare(x, y, p):\n",
    "    num_examples = len(x)\n",
    "\n",
    "    xx = np.zeros([num_examples, max_sent_size], dtype=int)\n",
    "    xx_mask = np.zeros([num_examples, max_sent_size], dtype=bool)\n",
    "    yy = np.zeros([num_examples], dtype=int)\n",
    "    pp = np.zeros([num_examples, max_sent_size], dtype=int)\n",
    "\n",
    "    for j in range(num_examples):\n",
    "        for i in range(max_sent_size):\n",
    "            if(i>=len(x[j])):\n",
    "                break\n",
    "            w = x[j][i]\n",
    "            xx[j][i] = word2id[w] if w in word2id else word2id['UNKNOWN_TOKEN']\n",
    "            xx_mask[j][i] = True\n",
    "            pp[j][i] = pos2id[p[j][i]]\n",
    "        yy[j] = sense2id[y[j]]\n",
    "        \n",
    "    return xx, xx_mask, yy, pp\n",
    "\n",
    "def model(xx, yy, mask, pp, train_cond=True):\n",
    "    num_batches = int(len(xx)/(batch_size*num_gpus))\n",
    "    _losses = 0\n",
    "    preds = []\n",
    "    preds_pos = []\n",
    "    true_pos = []\n",
    "    \n",
    "    for j in range(num_batches): \n",
    "        \n",
    "        s = j * batch_size * num_gpus\n",
    "        e = (j+1) * batch_size * num_gpus\n",
    "        xx_re = xx[s:e].reshape([num_gpus, batch_size, -1])\n",
    "        yy_re = yy[s:e].reshape([num_gpus, batch_size])\n",
    "        pp_re = pp[s:e].reshape([num_gpus, batch_size, -1])\n",
    "        mask_re = mask[s:e].reshape([num_gpus, batch_size, -1])\n",
    "        \n",
    "        \n",
    "        feed_dict = {x:xx_re, y:yy_re, y_pos:pp_re, x_mask:mask_re, is_train:train_cond, input_keep_prob:keep_prob, word_emb_mat:word_embedding}\n",
    "        \n",
    "        \n",
    "        if(train_cond==True):\n",
    "            _, _loss, step, _summary = sess.run([train_op, losses, global_step, summary], feed_dict)\n",
    "            summary_writer.add_summary(_summary, step)  \n",
    "#             print(\"Steps:{}\".format(step), \", Loss: {}\".format(_loss))\n",
    "        else:\n",
    "            _loss, pred, pred_pos = sess.run([losses, predictions, predictions_pos], feed_dict)\n",
    "            \n",
    "            for i in range(num_gpus):\n",
    "                preds.append(pred[i])\n",
    "            for nu in range(num_gpus):\n",
    "                for k in range(batch_size):\n",
    "                    preds_pos.append(pred_pos[nu][k][:sum(mask_re[nu][k])])\n",
    "                    true_pos.append(pp_re[nu][k][:sum(mask_re[nu][k])])\n",
    "\n",
    "        _losses = _losses + _loss\n",
    "\n",
    "    if(train_cond==False):\n",
    "        y_pred = []\n",
    "        pos_true = []\n",
    "        pos_pred = []\n",
    "        for pred in preds:\n",
    "            for bt in pred:\n",
    "                y_pred.append(bt)\n",
    "                \n",
    "        for i in range(len(preds_pos)):\n",
    "            for j in range(len(preds_pos[i])):\n",
    "                pos_pred.append(preds_pos[i][j])\n",
    "                pos_true.append(true_pos[i][j])\n",
    "\n",
    "        return _losses/num_batches, y_pred, pos_pred, pos_true\n",
    "    \n",
    "    return _losses/num_batches, step\n",
    "\n",
    "def eval_score(yy, pred, pos_true, pos_pred):\n",
    "    num_batches = int(len(yy)/(batch_size*num_gpus))\n",
    "    f1 = f1_score(yy[:batch_size*num_batches*num_gpus], pred, average='macro')\n",
    "    accu = accuracy_score(yy[:batch_size*num_batches*num_gpus], pred)\n",
    "    f1_pos = f1_score(pos_true, pos_pred, average='macro')\n",
    "    accu_pos = accuracy_score(pos_true, pos_pred)\n",
    "    return f1*100, accu*100, f1_pos*100, accu_pos*100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_id_train, mask_train, y_train, pos_id_train = data_prepare(x_train, label_train, pos_train)\n",
    "x_id_val, mask_val, y_val, pos_id_val = data_prepare(x_val, label_val,pos_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1 Step: 15 loss:22.9879 , Time: 152.1161\n",
      "Epoch: 2 Step: 30 loss:10.3295 , Time: 150.4417\n",
      "Epoch: 3 Step: 45 loss:6.6803 , Time: 154.9704\n",
      "Epoch: 4 Step: 60 loss:4.4384 , Time: 153.3356\n",
      "Epoch: 5 Step: 75 loss:3.4935 , Time: 147.7998\n",
      "Model Saved\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/users/btech/aviraj/cs771/lib/python3.5/site-packages/sklearn/metrics/classification.py:1137: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train: F1 Score:75.9903 Accuracy:76.5625  POS: F1 Score:70.6780 Accuracy:83.1291 Loss:3.1570 , Time: 102.7280\n",
      "Val: F1 Score:75.5346 Accuracy:76.7361  POS: F1 Score:70.3864 Accuracy:83.3135 Loss:3.2211 , Time: 20.9578\n",
      "Epoch: 6 Step: 90 loss:2.9307 , Time: 149.5268\n",
      "Epoch: 7 Step: 105 loss:2.6058 , Time: 150.6911\n",
      "Epoch: 8 Step: 120 loss:2.3255 , Time: 157.6312\n",
      "Epoch: 9 Step: 135 loss:2.1644 , Time: 151.4299\n",
      "Epoch: 10 Step: 150 loss:2.0037 , Time: 150.2264\n",
      "Model Saved\n",
      "Train: F1 Score:90.6329 Accuracy:91.3194  POS: F1 Score:79.5015 Accuracy:89.1729 Loss:1.8968 , Time: 104.0431\n",
      "Val: F1 Score:87.9316 Accuracy:89.0625  POS: F1 Score:79.7471 Accuracy:89.4258 Loss:1.9629 , Time: 21.5791\n",
      "Epoch: 11 Step: 165 loss:1.8450 , Time: 153.8003\n",
      "Epoch: 12 Step: 180 loss:1.7912 , Time: 150.7403\n",
      "Epoch: 13 Step: 195 loss:1.7029 , Time: 150.8080\n",
      "Epoch: 14 Step: 210 loss:1.6380 , Time: 152.2122\n",
      "Epoch: 15 Step: 225 loss:1.5591 , Time: 151.0603\n",
      "Model Saved\n",
      "Train: F1 Score:93.2014 Accuracy:93.6111  POS: F1 Score:83.1334 Accuracy:91.0912 Loss:1.5203 , Time: 105.4284\n",
      "Val: F1 Score:90.7023 Accuracy:91.8403  POS: F1 Score:82.2572 Accuracy:91.0932 Loss:1.6443 , Time: 21.4775\n",
      "Epoch: 16 Step: 240 loss:1.5222 , Time: 153.7953\n",
      "Epoch: 17 Step: 255 loss:1.4602 , Time: 149.3753\n",
      "Epoch: 18 Step: 270 loss:1.4519 , Time: 154.8704\n",
      "Epoch: 19 Step: 285 loss:1.4091 , Time: 151.5067\n",
      "Epoch: 20 Step: 300 loss:1.3779 , Time: 150.8597\n",
      "Model Saved\n",
      "Train: F1 Score:94.9014 Accuracy:95.2083  POS: F1 Score:84.1371 Accuracy:92.0702 Loss:1.3249 , Time: 103.5823\n",
      "Val: F1 Score:90.6616 Accuracy:91.4931  POS: F1 Score:83.1224 Accuracy:91.7482 Loss:1.5362 , Time: 20.4332\n",
      "Epoch: 21 Step: 315 loss:1.3514 , Time: 153.6794\n",
      "Epoch: 22 Step: 330 loss:1.3381 , Time: 151.3050\n",
      "Epoch: 23 Step: 345 loss:1.3219 , Time: 154.0576\n",
      "Epoch: 24 Step: 360 loss:1.3022 , Time: 150.1970\n",
      "Epoch: 25 Step: 375 loss:1.2786 , Time: 147.1968\n",
      "Model Saved\n",
      "Train: F1 Score:95.6367 Accuracy:95.7292  POS: F1 Score:85.7518 Accuracy:92.4563 Loss:1.2453 , Time: 102.1942\n",
      "Val: F1 Score:92.2990 Accuracy:93.2292  POS: F1 Score:83.7798 Accuracy:92.4543 Loss:1.4489 , Time: 21.2781\n",
      "Epoch: 26 Step: 390 loss:1.2318 , Time: 152.5874\n",
      "Epoch: 27 Step: 405 loss:1.2175 , Time: 150.6383\n",
      "Epoch: 28 Step: 420 loss:1.2014 , Time: 150.7890\n",
      "Epoch: 29 Step: 435 loss:1.1866 , Time: 150.7123\n",
      "Epoch: 30 Step: 450 loss:1.1857 , Time: 152.8568\n",
      "Model Saved\n",
      "Train: F1 Score:96.0412 Accuracy:96.4931  POS: F1 Score:85.6568 Accuracy:92.8685 Loss:1.1592 , Time: 102.3266\n",
      "Val: F1 Score:91.4996 Accuracy:92.3611  POS: F1 Score:85.0302 Accuracy:92.4883 Loss:1.4720 , Time: 19.8383\n",
      "Epoch: 31 Step: 465 loss:1.1601 , Time: 153.5217\n",
      "Epoch: 32 Step: 480 loss:1.1629 , Time: 150.1865\n",
      "Epoch: 33 Step: 495 loss:1.1335 , Time: 149.2082\n",
      "Epoch: 34 Step: 510 loss:1.1248 , Time: 153.0239\n",
      "Epoch: 35 Step: 525 loss:1.1069 , Time: 149.7498\n",
      "Model Saved\n",
      "Train: F1 Score:96.4278 Accuracy:96.6319  POS: F1 Score:87.1656 Accuracy:93.0685 Loss:1.0920 , Time: 104.7059\n",
      "Val: F1 Score:90.0173 Accuracy:90.9722  POS: F1 Score:85.1896 Accuracy:92.8584 Loss:1.4518 , Time: 20.7840\n",
      "Epoch: 36 Step: 540 loss:1.0926 , Time: 151.4367\n",
      "Epoch: 37 Step: 555 loss:1.0984 , Time: 150.3206\n",
      "Epoch: 38 Step: 570 loss:1.1087 , Time: 151.6847\n",
      "Epoch: 39 Step: 585 loss:1.0916 , Time: 151.6734\n",
      "Epoch: 40 Step: 600 loss:1.0809 , Time: 151.5601\n",
      "Model Saved\n",
      "Train: F1 Score:96.4744 Accuracy:96.7708  POS: F1 Score:87.8377 Accuracy:93.3619 Loss:1.0708 , Time: 104.7285\n",
      "Val: F1 Score:90.3115 Accuracy:91.3194  POS: F1 Score:85.7872 Accuracy:93.1774 Loss:1.4066 , Time: 20.6404\n",
      "Epoch: 41 Step: 615 loss:1.0712 , Time: 150.2466\n",
      "Epoch: 42 Step: 630 loss:1.0401 , Time: 139.9002\n",
      "Epoch: 43 Step: 645 loss:1.0551 , Time: 140.6543\n",
      "Epoch: 44 Step: 660 loss:1.0401 , Time: 134.9690\n",
      "Epoch: 45 Step: 675 loss:1.0318 , Time: 129.4382\n",
      "Model Saved\n",
      "Train: F1 Score:96.5315 Accuracy:96.7014  POS: F1 Score:88.5039 Accuracy:93.6984 Loss:1.0216 , Time: 92.3330\n",
      "Val: F1 Score:90.2184 Accuracy:90.7986  POS: F1 Score:85.5069 Accuracy:92.8924 Loss:1.4813 , Time: 18.6791\n",
      "Epoch: 46 Step: 690 loss:1.0376 , Time: 131.6116\n",
      "Epoch: 47 Step: 705 loss:1.0398 , Time: 153.8800\n",
      "Epoch: 48 Step: 720 loss:1.0364 , Time: 197.5601\n",
      "Epoch: 49 Step: 735 loss:1.0057 , Time: 198.9062\n",
      "Epoch: 50 Step: 750 loss:1.0140 , Time: 202.0647\n",
      "Model Saved\n",
      "Train: F1 Score:96.9378 Accuracy:97.1528  POS: F1 Score:88.1537 Accuracy:93.7957 Loss:1.0064 , Time: 150.3262\n",
      "Val: F1 Score:90.5247 Accuracy:91.3194  POS: F1 Score:87.2230 Accuracy:93.5389 Loss:1.3769 , Time: 32.3335\n",
      "Epoch: 51 Step: 765 loss:1.0081 , Time: 199.4464\n",
      "Epoch: 52 Step: 780 loss:0.9933 , Time: 203.7134\n",
      "Epoch: 53 Step: 795 loss:0.9885 , Time: 197.5780\n",
      "Epoch: 54 Step: 810 loss:0.9818 , Time: 199.6018\n",
      "Epoch: 55 Step: 825 loss:0.9849 , Time: 194.7120\n",
      "Model Saved\n",
      "Train: F1 Score:97.2323 Accuracy:97.3958  POS: F1 Score:88.8955 Accuracy:93.9127 Loss:0.9767 , Time: 151.5940\n",
      "Val: F1 Score:90.7803 Accuracy:91.8403  POS: F1 Score:87.7646 Accuracy:93.3986 Loss:1.3980 , Time: 32.5277\n",
      "Epoch: 56 Step: 840 loss:0.9821 , Time: 193.8034\n",
      "Epoch: 57 Step: 855 loss:0.9812 , Time: 195.9420\n",
      "Epoch: 58 Step: 870 loss:0.9501 , Time: 199.9444\n",
      "Epoch: 59 Step: 885 loss:0.9721 , Time: 201.9567\n",
      "Epoch: 60 Step: 900 loss:0.9580 , Time: 192.3138\n",
      "Model Saved\n",
      "Train: F1 Score:97.2169 Accuracy:97.3611  POS: F1 Score:88.6441 Accuracy:93.9193 Loss:0.9609 , Time: 156.8009\n",
      "Val: F1 Score:89.7416 Accuracy:90.9722  POS: F1 Score:86.0337 Accuracy:93.5729 Loss:1.3992 , Time: 32.9210\n"
     ]
    }
   ],
   "source": [
    "num_epochs = 60\n",
    "log_period = 5\n",
    "\n",
    "for i in range(num_epochs):\n",
    "    random = np.random.choice(len(y_train), size=(len(y_train)), replace=False)\n",
    "    x_id_train = x_id_train[random]\n",
    "    y_train = y_train[random]\n",
    "    mask_train = mask_train[random]    \n",
    "    pos_id_train = pos_id_train[random]\n",
    "\n",
    "    start_time = time.time()\n",
    "    train_loss, step = model(x_id_train, y_train, mask_train, pos_id_train)\n",
    "    time_taken = time.time() - start_time\n",
    "    print(\"Epoch:\", i+1,\"Step:\", step, \"loss:{0:.4f}\".format(train_loss), \", Time: {0:.4f}\".format(time_taken))\n",
    "    \n",
    "    if((i+1)%log_period==0):\n",
    "        saver.save(sess, save_path=save_dir)                         \n",
    "        print(\"Model Saved\")\n",
    "        start_time = time.time()\n",
    "        train_loss, train_pred, train_pred_pos, train_true_pos = model(x_id_train, y_train, mask_train, pos_id_train, train_cond=False)        \n",
    "        f1_, accu_, f1_pos_, accu_pos_ = eval_score(y_train, train_pred, train_pred_pos, train_true_pos)\n",
    "        time_taken = time.time() - start_time\n",
    "        print(\"Train: F1 Score:{0:.4f}\".format(f1_), \"Accuracy:{0:.4f}\".format(accu_), \" POS: F1 Score:{0:.4f}\".format(f1_pos_), \"Accuracy:{0:.4f}\".format(accu_pos_), \"Loss:{0:.4f}\".format(train_loss), \", Time: {0:.4f}\".format(time_taken))\n",
    "        \n",
    "        start_time = time.time()\n",
    "        val_loss, val_pred, val_pred_pos, val_true_pos = model(x_id_val, y_val, mask_val, pos_id_val, train_cond=False)        \n",
    "        f1_, accu_, f1_pos_, accu_pos_ = eval_score(y_val, val_pred, val_pred_pos, val_true_pos)\n",
    "        time_taken = time.time() - start_time\n",
    "        print(\"Val: F1 Score:{0:.4f}\".format(f1_), \"Accuracy:{0:.4f}\".format(accu_), \" POS: F1 Score:{0:.4f}\".format(f1_pos_), \"Accuracy:{0:.4f}\".format(accu_pos_), \"Loss:{0:.4f}\".format(val_loss), \", Time: {0:.4f}\".format(time_taken))\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "saver.restore(sess, save_dir)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "envs",
   "language": "python",
   "name": "cs771"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}