cnn_sentence_classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sentence Classification with Convolution Neural Networks\n",
    "Paper: Convolutional Neural Networks for Sentence Classification by Yoon Kim\n",
    "\n",
    "Dataset: http://cogcomp.cs.illinois.edu/Data/QA/QC/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# These are all the modules we'll be using later. Make sure you can import them\n",
    "# before proceeding further.\n",
    "%matplotlib inline\n",
    "from __future__ import print_function\n",
    "import collections\n",
    "import math\n",
    "import numpy as np\n",
    "import os\n",
    "import random\n",
    "import tensorflow as tf\n",
    "import zipfile\n",
    "from matplotlib import pylab\n",
    "from six.moves import range\n",
    "from six.moves.urllib.request import urlretrieve\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading the dataset\n",
    "This dataset is composed of questions as inputs and their respective type as the output. For example, (e.g. Who was Abraham Lincon?) and the output or label would be Human.\n",
    "\n",
    "Dataset: http://cogcomp.org/Data/QA/QC/\n",
    "\n",
    "For the moment you will have to visit this link"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files found and verified.\n"
     ]
    }
   ],
   "source": [
    "global dir_name,filenames,num_files\n",
    "\n",
    "dir_name = 'question-classif-data'\n",
    "url= 'http://cogcomp.org/Data/QA/QC/'\n",
    "\n",
    "if not os.path.exists(dir_name):\n",
    "    os.mkdir(dir_name)\n",
    "\n",
    "filenames = ['trec-train-1000.txt','trec-test.txt']\n",
    "num_files = 2\n",
    "\n",
    "if not os.path.isfile(os.path.join(dir_name,filenames[i])):\n",
    "    try:\n",
    "        print('Downloading training and test datasets')\n",
    "        urlretrieve(url+\"train_1000.label\", os.path.join(dir_name,\"trec-train-1000.txt\"))\n",
    "        urlretrieve(url+\"TREC_10.label\", os.path.join(dir_name,\"trec-test.txt\"))\n",
    "        print('Download successful')\n",
    "    except:\n",
    "        print('Something went wrong while downloading data. \\n'+\n",
    "        'Can you manually download data and place them in the question-classif-data folder?\\n'+\n",
    "        'Train: http://cogcomp.org/Data/QA/QC/train_1000.label as trec-train-1000.txt\\n'+\n",
    "        'Test: http://cogcomp.org/Data/QA/QC/TREC_10.label as trec-test.txt')\n",
    "    \n",
    "\n",
    "for i in range(len(filenames)):\n",
    "    file_exists = os.path.isfile(os.path.join(dir_name,filenames[i]))\n",
    "    assert file_exists\n",
    "\n",
    "print('Files found and verified.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Format raw data into readable the data\n",
    "\n",
    "Here we read data from files.\n",
    "Calculate the maximum sentence length in the database\n",
    "Then split each sentence into a sequence of words\n",
    "Pad each sentence with \"PAD\" until the length of the sentence is the maximum length observed earlier."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Processing file question-classif-data/trec-train-1000.txt\n",
      "\tQuestion 0: ['manner', 'how', 'did', 'serfdom', 'develop', 'in', 'and', 'then', 'leave', 'russia', '?']\n",
      "\tLabel 0: DESC\n",
      "\n",
      "\tQuestion 1: ['cremat', 'what', 'films', 'featured', 'the', 'character', 'popeye', 'doyle', '?']\n",
      "\tLabel 1: ENTY\n",
      "\n",
      "\tQuestion 2: ['manner', 'how', 'can', 'i', 'find', 'a', 'list', 'of', 'celebrities', \"'\", 'real', 'names', '?']\n",
      "\tLabel 2: DESC\n",
      "\n",
      "\tQuestion 3: ['animal', 'what', 'fowl', 'grabs', 'the', 'spotlight', 'after', 'the', 'chinese', 'year', 'of', 'the', 'monkey', '?']\n",
      "\tLabel 3: ENTY\n",
      "\n",
      "\tQuestion 4: ['exp', 'what', 'is', 'the', 'full', 'form', 'of', '.com', '?']\n",
      "\tLabel 4: ABBR\n",
      "\n",
      "\n",
      "Processing file question-classif-data/trec-test.txt\n",
      "\tQuestion 0: ['manner', 'how', 'did', 'serfdom', 'develop', 'in', 'and', 'then', 'leave', 'russia', '?']\n",
      "\tLabel 0: DESC\n",
      "\n",
      "\tQuestion 1: ['cremat', 'what', 'films', 'featured', 'the', 'character', 'popeye', 'doyle', '?']\n",
      "\tLabel 1: ENTY\n",
      "\n",
      "\tQuestion 2: ['manner', 'how', 'can', 'i', 'find', 'a', 'list', 'of', 'celebrities', \"'\", 'real', 'names', '?']\n",
      "\tLabel 2: DESC\n",
      "\n",
      "\tQuestion 3: ['animal', 'what', 'fowl', 'grabs', 'the', 'spotlight', 'after', 'the', 'chinese', 'year', 'of', 'the', 'monkey', '?']\n",
      "\tLabel 3: ENTY\n",
      "\n",
      "\tQuestion 4: ['exp', 'what', 'is', 'the', 'full', 'form', 'of', '.com', '?']\n",
      "\tLabel 4: ABBR\n",
      "\n",
      "Max Sentence Length: 33\n",
      "\n",
      "Normalizing all sentences to same length\n",
      "\tTrain questions normalized\n",
      "\tTest questions normalized\n",
      "\t\tSample test question: %s ['dist', 'how', 'far', 'is', 'it', 'from', 'denver', 'to', 'aspen', '?', 'PAD', 'PAD', 'PAD', 'PAD', 'PAD', 'PAD', 'PAD', 'PAD', 'PAD', 'PAD', 'PAD', 'PAD', 'PAD', 'PAD', 'PAD', 'PAD', 'PAD', 'PAD', 'PAD', 'PAD', 'PAD', 'PAD', 'PAD']\n"
     ]
    }
   ],
   "source": [
    "max_sent_length = 0\n",
    "def read_data(filename):\n",
    "  global max_sent_length\n",
    "  questions = []\n",
    "  labels = []\n",
    "  with open(filename,'r',encoding='latin-1') as f:        \n",
    "    for row in f:\n",
    "        row_str = row.split(\":\")\n",
    "        lb,q = row_str[0],row_str[1]\n",
    "        q = q.lower()\n",
    "        labels.append(lb)\n",
    "        questions.append(q.split())        \n",
    "        if len(questions[-1])>max_sent_length:\n",
    "            max_sent_length = len(questions[-1])\n",
    "  return questions,labels\n",
    "\n",
    "\n",
    "global train_questions,train_labels\n",
    "global test_questions,test_labels\n",
    "\n",
    "for i in range(num_files):    \n",
    "    print('\\nProcessing file %s'%os.path.join(dir_name,filenames[i]))\n",
    "    if i==0:\n",
    "        train_questions,train_labels = read_data(os.path.join(dir_name,filenames[i]))\n",
    "        assert len(train_questions)==len(train_labels)\n",
    "    elif i==1:\n",
    "        test_questions,test_labels = read_data(os.path.join(dir_name,filenames[i]))\n",
    "        assert len(test_questions)==len(test_labels)\n",
    "    for j in range(5):\n",
    "        print('\\tQuestion %d: %s' %(j,train_questions[j]))\n",
    "        print('\\tLabel %d: %s\\n'%(j,train_labels[j]))\n",
    "        \n",
    "print('Max Sentence Length: %d'%max_sent_length)\n",
    "print('\\nNormalizing all sentences to same length')\n",
    "\n",
    "for qi,que in enumerate(train_questions):\n",
    "    for _ in range(max_sent_length-len(que)):\n",
    "        que.append('PAD')\n",
    "    assert len(que)==max_sent_length\n",
    "    train_questions[qi] = que\n",
    "print('\\tTrain questions normalized')\n",
    "for qi,que in enumerate(test_questions):\n",
    "    for _ in range(max_sent_length-len(que)):\n",
    "        que.append('PAD')\n",
    "    assert len(que)==max_sent_length\n",
    "    test_questions[qi] = que\n",
    "print('\\tTest questions normalized')  \n",
    "print('\\t\\tSample test question: %s',test_questions[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Make data Numerical\n",
    "\n",
    "Here we convert those textual sequences of words into actual neumerical sequences by assigning each unique word an ID."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "49500 Words found.\n",
      "Found 3369 words in the vocabulary. \n",
      "All words (count) [('PAD', 34407), ('?', 1454), ('the', 999), ('what', 963), ('is', 587)]\n",
      "0th entry in dictionary: %s PAD\n",
      "\n",
      "Sample data\n",
      "[38, 12, 19, 1180, 2159, 6, 28, 2907, 1354, 618, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
      "[44, 3, 697, 2308, 2, 183, 1794, 2076, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
      "\n",
      "Vocabulary size:  3369\n",
      "\n",
      "Train size:  1000\n",
      "Test size:  500\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "def build_dataset(questions):\n",
    "    words = []\n",
    "    data_list = []\n",
    "    count = []\n",
    "    for d in questions:\n",
    "        words.extend(d)\n",
    "    print('%d Words found.'%len(words))    \n",
    "    print('Found %d words in the vocabulary. '%len(collections.Counter(words).most_common()))\n",
    "    count.extend(collections.Counter(words).most_common())\n",
    "    dictionary = dict()\n",
    "    for word, _ in count:\n",
    "        dictionary[word] = len(dictionary)\n",
    "        \n",
    "    for d in questions:\n",
    "        data = list()\n",
    "        for word in d:\n",
    "            index = dictionary[word]        \n",
    "            data.append(index)\n",
    "            \n",
    "        data_list.append(data)\n",
    "    reverse_dictionary = dict(zip(dictionary.values(), dictionary.keys())) \n",
    "    return data_list, count, dictionary, reverse_dictionary\n",
    "\n",
    "global data_list, count, dictionary, reverse_dictionary\n",
    "\n",
    "all_questions = list(train_questions)\n",
    "all_questions.extend(test_questions)\n",
    "\n",
    "all_question_ind, count, dictionary, reverse_dictionary = build_dataset(all_questions)\n",
    "\n",
    "print('All words (count)', count[:5])\n",
    "print('0th entry in dictionary: %s',reverse_dictionary[0])\n",
    "print('\\nSample data') \n",
    "print(all_question_ind[0])\n",
    "print(all_question_ind[1])\n",
    "\n",
    "print('\\nVocabulary size: ',len(dictionary))\n",
    "vocabulary_size = len(dictionary)\n",
    "\n",
    "print('\\nTrain size: ',len(train_questions))\n",
    "print('Test size: ',len(test_questions))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create a function that outputs a batch of data when challed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sample batch labels\n",
      "[3 4 3 4 5 2 2 2 3 2 0 3 2 2 4 1]\n",
      "[3 0 3 3 0 4 2 3 3 4 2 1 4 1 5 4]\n"
     ]
    }
   ],
   "source": [
    "batch_size = 16\n",
    "sent_length = 33\n",
    "num_classes = 6\n",
    "all_labels = ['NUM','LOC','HUM','DESC','ENTY','ABBR'] # All the types of question that are in the dataset\n",
    "\n",
    "class BatchGenerator(object):\n",
    "    def __init__(self,batch_size,questions,labels):\n",
    "        self.questions = questions\n",
    "        self.labels = labels\n",
    "        self.text_size = len(questions)\n",
    "        self.batch_size = batch_size\n",
    "        self.data_index = 0\n",
    "        assert len(self.questions)==len(self.labels)\n",
    "        \n",
    "    def generate_batch(self):\n",
    "        global sent_length,num_classes\n",
    "        global dictionary\n",
    "        inputs = np.zeros((self.batch_size,sent_length,vocabulary_size),dtype=np.float32)\n",
    "        labels_ohe = np.zeros((self.batch_size,num_classes),dtype=np.float32)\n",
    "        \n",
    "        if self.data_index + self.batch_size >= self.text_size:\n",
    "            self.data_index = 0\n",
    "            \n",
    "        for qi,que in enumerate(self.questions[self.data_index:self.data_index+self.batch_size]):\n",
    "            for wi,word in enumerate(que):\n",
    "                #q_arr[qi*batch_size + wi] = dictionary[word]\n",
    "                inputs[qi,wi,dictionary[word]] = 1.0\n",
    "            \n",
    "            #print(self.data_index,qi,len(self.labels))\n",
    "            labels_ohe[qi,all_labels.index(self.labels[self.data_index + qi])] = 1.0\n",
    "            #print('%s'%([w for w in que]),self.labels[self.data_index+qi])            \n",
    "            \n",
    "        self.data_index = (self.data_index + self.batch_size)%self.text_size\n",
    "            \n",
    "        return inputs,labels_ohe\n",
    "    \n",
    "    def return_index(self):\n",
    "        return self.data_index\n",
    "    \n",
    "sample_gen = BatchGenerator(batch_size,train_questions,train_labels)\n",
    "sample_batch_inputs,sample_batch_labels = sample_gen.generate_batch()\n",
    "sample_batch_inputs_2,sample_batch_labels_2 = sample_gen.generate_batch()\n",
    "assert np.all(np.asarray([dictionary[w] for w in train_questions[0]],dtype=np.int32) \n",
    "              == np.argmax(sample_batch_inputs[0,:,:],axis=1))\n",
    "print('Sample batch labels')\n",
    "print(np.argmax(sample_batch_labels,axis=1))\n",
    "print(np.argmax(sample_batch_labels_2,axis=1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sentence Classifying Convolution Neural Network (Tensorflow)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "batch_size = 16\n",
    "\n",
    "# inputs and labels\n",
    "sent_inputs = tf.placeholder(shape=[batch_size,sent_length,vocabulary_size],dtype=tf.float32,name='sentence_inputs')\n",
    "sent_labels = tf.placeholder(shape=[batch_size,num_classes],dtype=tf.float32,name='sentence_labels')\n",
    "\n",
    "# 3 filters with different context window sizes (3,5,7)\n",
    "# Each of this filter spans the full one-hot-encoded length of each word and the context window width\n",
    "w1 = tf.Variable(tf.truncated_normal([3,vocabulary_size,1],stddev=0.02,dtype=tf.float32),name='weights_1')\n",
    "b1 = tf.Variable(tf.random_uniform([1],0,0.01,dtype=tf.float32),name='bias_1')\n",
    "\n",
    "w2 = tf.Variable(tf.truncated_normal([5,vocabulary_size,1],stddev=0.02,dtype=tf.float32),name='weights_2')\n",
    "b2 = tf.Variable(tf.random_uniform([1],0,0.01,dtype=tf.float32),name='bias_2')\n",
    "\n",
    "w3 = tf.Variable(tf.truncated_normal([7,vocabulary_size,1],stddev=0.02,dtype=tf.float32),name='weights_3')\n",
    "b3 = tf.Variable(tf.random_uniform([1],0,0.01,dtype=tf.float32),name='bias_3')\n",
    "\n",
    "# Calculate the output for all the filters with a stride 1\n",
    "h1_1 = tf.nn.tanh(tf.nn.conv1d(sent_inputs,w1,stride=1,padding='SAME') + b1)\n",
    "h1_2 = tf.nn.tanh(tf.nn.conv1d(sent_inputs,w2,stride=1,padding='SAME') + b2)\n",
    "h1_3 = tf.nn.tanh(tf.nn.conv1d(sent_inputs,w3,stride=1,padding='SAME') + b3)\n",
    "\n",
    "# This is doing the max pooling. Thereare two options to do the max pooling\n",
    "# 1. Use tf.nn.max_pool operation on a tensor made by concatenating h1_1,h1_2,h1_3 and converting that tensor to 4D\n",
    "# (Because max_pool takes a tensor of rank >= 4 )\n",
    "# 2. Do the max pooling separately for each filter output and combine them using tf.concat \n",
    "# (this is the one used in the code)\n",
    "\n",
    "h2_1 = tf.reduce_max(h1_1,axis=1)\n",
    "h2_2 = tf.reduce_max(h1_2,axis=1)\n",
    "h2_3 = tf.reduce_max(h1_3,axis=1)\n",
    "\n",
    "h2 = tf.concat([h2_1,h2_2,h2_3],axis=1)\n",
    "h2_shape = h2.get_shape().as_list()\n",
    "\n",
    "# Weights and bias of the output layer\n",
    "w_fc1 = tf.Variable(tf.truncated_normal([h2_shape[1],num_classes],stddev=0.005,dtype=tf.float32),name='weights_fulcon_1')\n",
    "b_fc1 = tf.Variable(tf.random_uniform([num_classes],0,0.01,dtype=tf.float32),name='bias_fulcon_1')\n",
    "\n",
    "# since h2 is 2d [batch_size,output_width] reshaping the output is not required as it usually do in CNNs\n",
    "logits = tf.matmul(h2,w_fc1) + b_fc1\n",
    "\n",
    "predictions = tf.argmax(tf.nn.softmax(logits),axis=1)\n",
    "\n",
    "# Loss (Cross-Entropy)\n",
    "loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=sent_labels,logits=logits))\n",
    "\n",
    "# Momentum Optimizer\n",
    "optimizer = tf.train.MomentumOptimizer(learning_rate=0.01,momentum=0.9).minimize(loss)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized\n",
      "\n",
      "Train Loss at Epoch 0: 1.74\n",
      "Test accuracy at Epoch 0: 18.548\n",
      "Train Loss at Epoch 1: 1.68\n",
      "Test accuracy at Epoch 1: 18.548\n",
      "Train Loss at Epoch 2: 1.67\n",
      "Test accuracy at Epoch 2: 18.548\n",
      "Train Loss at Epoch 3: 1.66\n",
      "Test accuracy at Epoch 3: 18.548\n",
      "Train Loss at Epoch 4: 1.66\n",
      "Test accuracy at Epoch 4: 18.548\n",
      "Train Loss at Epoch 5: 1.65\n",
      "Test accuracy at Epoch 5: 18.548\n",
      "Train Loss at Epoch 6: 1.63\n",
      "Test accuracy at Epoch 6: 30.040\n",
      "Train Loss at Epoch 7: 1.55\n",
      "Test accuracy at Epoch 7: 30.242\n",
      "Train Loss at Epoch 8: 1.42\n",
      "Test accuracy at Epoch 8: 30.444\n",
      "Train Loss at Epoch 9: 1.32\n",
      "Test accuracy at Epoch 9: 31.452\n",
      "Train Loss at Epoch 10: 1.25\n",
      "Test accuracy at Epoch 10: 31.452\n",
      "Train Loss at Epoch 11: 1.22\n",
      "Test accuracy at Epoch 11: 31.452\n",
      "Train Loss at Epoch 12: 1.20\n",
      "Test accuracy at Epoch 12: 31.452\n",
      "Train Loss at Epoch 13: 1.19\n",
      "Test accuracy at Epoch 13: 31.452\n",
      "Train Loss at Epoch 14: 1.17\n",
      "Test accuracy at Epoch 14: 31.452\n",
      "Train Loss at Epoch 15: 1.17\n",
      "Test accuracy at Epoch 15: 31.452\n",
      "Train Loss at Epoch 16: 1.16\n",
      "Test accuracy at Epoch 16: 31.653\n",
      "Train Loss at Epoch 17: 1.16\n",
      "Test accuracy at Epoch 17: 31.653\n",
      "Train Loss at Epoch 18: 1.15\n",
      "Test accuracy at Epoch 18: 31.653\n",
      "Train Loss at Epoch 19: 1.15\n",
      "Test accuracy at Epoch 19: 31.653\n",
      "Train Loss at Epoch 20: 1.14\n",
      "Test accuracy at Epoch 20: 31.653\n",
      "Train Loss at Epoch 21: 1.14\n",
      "Test accuracy at Epoch 21: 31.653\n",
      "Train Loss at Epoch 22: 1.14\n",
      "Test accuracy at Epoch 22: 31.653\n",
      "Train Loss at Epoch 23: 1.14\n",
      "Test accuracy at Epoch 23: 31.653\n",
      "Train Loss at Epoch 24: 1.13\n",
      "Test accuracy at Epoch 24: 31.653\n",
      "Train Loss at Epoch 25: 1.13\n",
      "Test accuracy at Epoch 25: 31.653\n",
      "Train Loss at Epoch 26: 1.13\n",
      "Test accuracy at Epoch 26: 31.653\n",
      "Train Loss at Epoch 27: 1.12\n",
      "Test accuracy at Epoch 27: 31.653\n",
      "Train Loss at Epoch 28: 1.12\n",
      "Test accuracy at Epoch 28: 31.653\n",
      "Train Loss at Epoch 29: 1.12\n",
      "Test accuracy at Epoch 29: 33.266\n",
      "Train Loss at Epoch 30: 1.12\n",
      "Test accuracy at Epoch 30: 56.653\n",
      "Train Loss at Epoch 31: 1.12\n",
      "Test accuracy at Epoch 31: 56.452\n",
      "Train Loss at Epoch 32: 1.11\n",
      "Test accuracy at Epoch 32: 55.847\n",
      "Train Loss at Epoch 33: 1.11\n",
      "Test accuracy at Epoch 33: 55.444\n",
      "Train Loss at Epoch 34: 1.10\n",
      "Test accuracy at Epoch 34: 55.242\n",
      "Train Loss at Epoch 35: 1.09\n",
      "Test accuracy at Epoch 35: 55.847\n",
      "Train Loss at Epoch 36: 1.07\n",
      "Test accuracy at Epoch 36: 57.258\n",
      "Train Loss at Epoch 37: 1.04\n",
      "Test accuracy at Epoch 37: 59.879\n",
      "Train Loss at Epoch 38: 0.99\n",
      "Test accuracy at Epoch 38: 65.121\n",
      "Train Loss at Epoch 39: 0.92\n",
      "Test accuracy at Epoch 39: 69.153\n",
      "Train Loss at Epoch 40: 0.84\n",
      "Test accuracy at Epoch 40: 70.565\n"
     ]
    }
   ],
   "source": [
    "# With filter widths [3,5,7] the algorithm achieves around ~90% accuracy on test dataset (50 epochs). \n",
    "\n",
    "session = tf.InteractiveSession()\n",
    "\n",
    "batch_size = 16\n",
    "num_steps = 50\n",
    "tf.global_variables_initializer().run()\n",
    "train_gen = BatchGenerator(batch_size,train_questions,train_labels)\n",
    "test_gen = BatchGenerator(batch_size,test_questions,test_labels)\n",
    "\n",
    "test_interval = 1\n",
    "\n",
    "# Get accuracy \n",
    "def accuracy(labels,preds):\n",
    "    return np.sum(np.argmax(labels,axis=1)==preds)/labels.shape[0]\n",
    "\n",
    "print('Initialized\\n')\n",
    "\n",
    "\n",
    "for step in range(num_steps):\n",
    "    avg_loss = []\n",
    "    for tr_i in range((len(train_questions)//batch_size)-1):\n",
    "        tr_inputs, tr_labels = train_gen.generate_batch()\n",
    "        \n",
    "        l,_ = session.run([loss,optimizer],feed_dict={sent_inputs: tr_inputs, sent_labels: tr_labels})\n",
    "        avg_loss.append(l)\n",
    "        \n",
    "    print('Train Loss at Epoch %d: %.2f'%(step,np.mean(avg_loss)))\n",
    "    test_accuracy = []\n",
    "    if (step+1)%test_interval==0:        \n",
    "        for ts_i in range((len(test_questions)-1)//batch_size):\n",
    "            ts_inputs,ts_labels = test_gen.generate_batch()\n",
    "            preds = session.run(predictions,feed_dict={sent_inputs: ts_inputs, sent_labels: ts_labels})\n",
    "            test_accuracy.append(accuracy(ts_labels,preds))\n",
    "            \n",
    "        print('Test accuracy at Epoch %d: %.3f'%(step,np.mean(test_accuracy)*100.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}