Word_Prediction_using_Interpolated_Knesser_Ney.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# <u>Word prediction</u> \n",
    "### Language Model based on n-gram Probabilistic Model\n",
    "### Interpolated Knesser Ney Used\n",
    "### Highest Order n-gram used is Quadgram\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## <u>Import corpus</u>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.util import ngrams\n",
    "from collections import defaultdict\n",
    "from collections import OrderedDict\n",
    "import string\n",
    "import time\n",
    "import gc\n",
    "from math import log10\n",
    "start_time = time.time()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## <u>Do preprocessing</u>:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Remove the punctuations and lowercase the tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#returns: string\n",
    "#arg: string\n",
    "#remove punctuations, change to lowercase ,retain the apostrophe mark\n",
    "def removePunctuations(sen):\n",
    "    #split the string into word tokens\n",
    "    temp_l = sen.split()\n",
    "    #print(temp_l)\n",
    "    i = 0\n",
    "    j = 0\n",
    "    \n",
    "    #changes the word to lowercase and removes punctuations from it\n",
    "    for word in temp_l :\n",
    "        j = 0\n",
    "        #print(len(word))\n",
    "        for l in word :\n",
    "            if l in string.punctuation:\n",
    "                if l == \"'\":\n",
    "                    if j+1<len(word) and word[j+1] == 's':\n",
    "                        j = j + 1\n",
    "                        continue\n",
    "                word = word.replace(l,\" \")\n",
    "                #print(j,word[j])\n",
    "            j += 1\n",
    "\n",
    "        temp_l[i] = word.lower()\n",
    "        i=i+1   \n",
    "\n",
    "    #spliting is being done here beacause in sentences line here---so after punctuation removal it should \n",
    "    #become \"here so\"   \n",
    "    content = \" \".join(temp_l)\n",
    "\n",
    "    return content"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tokenize and load the corpus data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#returns : void\n",
    "#arg: string,dict,dict,dict,dict\n",
    "#loads the corpus for the dataset and makes the frequency count of quadgram ,bigram and trigram strings\n",
    "def loadCorpus(file_path, bi_dict, tri_dict, quad_dict, vocab_dict):\n",
    "\n",
    "    w1 = ''    #for storing the 3rd last word to be used for next token set\n",
    "    w2 = ''    #for storing the 2nd last word to be used for next token set\n",
    "    w3 = ''    #for storing the last word to be used for next token set\n",
    "    token = []\n",
    "    #total no. of words in the corpus\n",
    "    word_len = 0\n",
    "\n",
    "    #open the corpus file and read it line by line\n",
    "    with open(file_path,'r') as file:\n",
    "        for line in file:\n",
    "\n",
    "            #split the string into word tokens\n",
    "            temp_l = line.split()\n",
    "            i = 0\n",
    "            j = 0\n",
    "            \n",
    "            #does the same as the removePunctuations() function,implicit declratation for performance reasons\n",
    "            #changes the word to lowercase and removes punctuations from it\n",
    "            for word in temp_l :\n",
    "                j = 0\n",
    "                #print(len(word))\n",
    "                for l in word :\n",
    "                    if l in string.punctuation:\n",
    "                        if l == \"'\":\n",
    "                            if j+1<len(word) and word[j+1] == 's':\n",
    "                                j = j + 1\n",
    "                                continue\n",
    "                        word = word.replace(l,\" \")\n",
    "                        #print(j,word[j])\n",
    "                    j += 1\n",
    "\n",
    "                temp_l[i] = word.lower()\n",
    "                i=i+1   \n",
    "\n",
    "            #spliting is being done here beacause in sentences line here---so after punctuation removal it should \n",
    "            #become \"here so\"   \n",
    "            content = \" \".join(temp_l)\n",
    "\n",
    "            token = content.split()\n",
    "            word_len = word_len + len(token)  \n",
    "\n",
    "            if not token:\n",
    "                continue\n",
    "\n",
    "            #add the last word from previous line\n",
    "            if w3!= '':\n",
    "                token.insert(0,w3)\n",
    "\n",
    "            temp0 = list(ngrams(token,2))\n",
    "\n",
    "            #since we are reading line by line some combinations of word might get missed for pairing\n",
    "            #for trigram\n",
    "            #first add the previous words\n",
    "            if w2!= '':\n",
    "                token.insert(0,w2)\n",
    "\n",
    "            #tokens for trigrams\n",
    "            temp1 = list(ngrams(token,3))\n",
    "\n",
    "            #insert the 3rd last word from previous line for quadgram pairing\n",
    "            if w1!= '':\n",
    "                token.insert(0,w1)\n",
    "\n",
    "            #add new unique words to the vocaulary set if available\n",
    "            for word in token:\n",
    "                if word not in vocab_dict:\n",
    "                    vocab_dict[word] = 1\n",
    "                else:\n",
    "                    vocab_dict[word]+= 1\n",
    "                  \n",
    "            #tokens for quadgrams\n",
    "            temp2 = list(ngrams(token,4))\n",
    "\n",
    "            #count the frequency of the bigram sentences\n",
    "            for t in temp0:\n",
    "                sen = ' '.join(t)\n",
    "                bi_dict[sen] += 1\n",
    "\n",
    "            #count the frequency of the trigram sentences\n",
    "            for t in temp1:\n",
    "                sen = ' '.join(t)\n",
    "                tri_dict[sen] += 1\n",
    "\n",
    "            #count the frequency of the quadgram sentences\n",
    "            for t in temp2:\n",
    "                sen = ' '.join(t)\n",
    "                quad_dict[sen] += 1\n",
    "\n",
    "\n",
    "            #then take out the last 3 words\n",
    "            n = len(token)\n",
    "\n",
    "            #store the last few words for the next sentence pairing\n",
    "            if (n -3) >= 0:\n",
    "                w1 = token[n -3]\n",
    "            if (n -2) >= 0:\n",
    "                w2 = token[n -2]\n",
    "            if (n -1) >= 0:\n",
    "                w3 = token[n -1]\n",
    "    return word_len"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Some Misc functions for debugging"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sort the probable words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#for sorting the probable word acc. to their probabilities\n",
    "\n",
    "#returns: void\n",
    "#arg: dict\n",
    "def sortProbWordDict(prob_dict):\n",
    "   \n",
    "    for key in prob_dict:\n",
    "        if len(prob_dict[key])>1:\n",
    "            #only at most top 2 most probable words have been taken\n",
    "            prob_dict[key] = sorted(prob_dict[key],reverse = True)[:2]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## <u>Driver function for doing the prediction</u>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#returns: string\n",
    "#arg: string,dict\n",
    "#does prediction for the the sentence\n",
    "def doPrediction(sen, prob_dict):\n",
    "    if sen in prob_dict:\n",
    "        return prob_dict[sen][0][1]\n",
    "    else:\n",
    "        return \"\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## <u>For Taking input from the User</u>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#returns: string\n",
    "#arg: void\n",
    "#for taking input from user\n",
    "def takeInput():\n",
    "    cond = False\n",
    "    #take input\n",
    "    while(cond == False):\n",
    "        sen = input('Enter the string\\n')\n",
    "        sen = removePunctuations(sen)\n",
    "        temp = sen.split()\n",
    "        if len(temp) < 3:\n",
    "            print(\"Please enter atleast 3 words !\")\n",
    "        else:\n",
    "            cond = True\n",
    "            temp = temp[-3:]\n",
    "    sen = \" \".join(temp)\n",
    "    return sen"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## <u>Test Score ,Perplexity Calculation:</u>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### For computing the Test Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#return:int\n",
    "#arg:list,dict,dict,dict,dict\n",
    "#computes the score for test data\n",
    "def computeTestScore(test_sent,tri_dict,quad_dict,vocab_dict,prob_dict):\n",
    "    #increment the score value if correct prediction is made else decrement its value\n",
    "    score = 0\n",
    "    wrong = 0\n",
    "    total = 0\n",
    "    with open('Test_Scores/Knesser_Ney_Interpolated_Score.txt','w') as w:\n",
    "        for sent in test_sent:\n",
    "            sen_token = sent[:3]\n",
    "            sen = \" \".join(sen_token)\n",
    "            correct_word = sent[3]\n",
    "           \n",
    "            result = doPrediction(sen,prob_dict)\n",
    "            if result == correct_word:\n",
    "                score+=1\n",
    "            else:\n",
    "                wrong += 1\n",
    "\n",
    "            total += 1\n",
    "            \n",
    "        w.write('Total Word Prdictions: '+str(total) + '\\n' +'Correct Prdictions: '+str(score) +\n",
    "                '\\n'+'Wrong Prdictions: '+str(wrong) + '\\n'+'ACCURACY: '+str((score/total)*100)+'%' )\n",
    "        #print stats\n",
    "        print('Total Word Prdictions: '+str(total) + '\\n' +'Correct Prdictions: '+str(score) +\n",
    "                '\\n'+'Wrong Prdictions: '+str(wrong) + '\\n'+'ACCURACY:'+str((score/total)*100)+'%' )\n",
    "    return score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### For Computing the Perplexity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#return:float\n",
    "#arg:list,int,dict,dict,dict,dict\n",
    "#computes the score for test data\n",
    "def computePerplexity(test_quadgrams,token_len,tri_dict,quad_dict,vocab_dict,prob_dict):\n",
    "    \n",
    "    perplexity = float(1.0)\n",
    "    n = token_len\n",
    "\n",
    "    for item in quad_dict:\n",
    "        sen_token = item.split()\n",
    "        sen = ' '.join(sen_token[0:3])\n",
    "        prob = quad_dict[item]/tri_dict[sen]\n",
    "        perplexity = perplexity * ( prob**(1./n))\n",
    "    with open('Test_Scores/Knesser_Ney_Interpolated_Score.txt','a') as w:\n",
    "        w.write('\\nPerplexity: '+str(perplexity))\n",
    "    return perplexity"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Interpolated Knesser Ney related Work"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### For creating Dictionaries required for Knesser Ney Probability Calculation\n",
    "This function makes dictionaries which keeps count of ngrams where Wn is followed by Wn-1 and also count of ngrams where Wn-1 preceeds the ngram "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#creates the dictionaries required for computing Interpolated Knesser Ney probability\n",
    "#return:dict,dict\n",
    "#arg:dict, int\n",
    "def createKNDict(ngram_dict, n):\n",
    "    #for knesser ney probability formula we need to find to important things \n",
    "    #first is for P(Wn|Wn-1) if find no. of ngrams which ends with Wn and no. of ngrams which starts \n",
    "    #with Wn-1\n",
    "    #so we divide the formula into two parts ,first part can be found in constant time\n",
    "    #and second term is found here\n",
    "    i = 0\n",
    "    d = 0.75\n",
    "    #for storing count of ngram ending with Wn,key:unigram\n",
    "    first_dict = {}\n",
    "    #for storing count of ngram having Wn-1 as its starting part, key: trigram sentence\n",
    "    sec_dict = {}\n",
    "    \n",
    "    for key in ngram_dict:\n",
    "        #split the key sentence into tokens \n",
    "        ngram_token = key.split()\n",
    "        #since the indexing is from 0 ,so for quadgram we need to create a sentence of three words\n",
    "        #so start from 0 to 2,so we subtract 1,similarly for trigram from 0 to 1 \n",
    "        n_1gram_sen = ' '.join(ngram_token[:n-1])\n",
    "        \n",
    "        #n_1gram_sen is the word that  stars in sec_dict[n_1gram_sen] number of times in ngram_dict \n",
    "        if n_1gram_sen not in sec_dict:\n",
    "            sec_dict[ n_1gram_sen ] = 1\n",
    "        else:\n",
    "            sec_dict[ n_1gram_sen ] += 1\n",
    "            \n",
    "        if ngram_token[-1] not in first_dict:\n",
    "            first_dict[ ngram_token[-1] ] = 1\n",
    "        else:\n",
    "            first_dict[ ngram_token[-1] ] += 1\n",
    "    \n",
    "    return first_dict, sec_dict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### For Computing Knesser Ney probability for various words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Finds the Knesser Ney probability for prediction\n",
    "#return:dict,dict\n",
    "#arg:dict, int\n",
    "def computeKnesserNeyProb(vocab_dict, bi_dict, tri_dict, quad_dict, prob_dict ):\n",
    "        d = 0.75\n",
    "        #first create the dict for storing the count of Wn-1 followed by Wn and for\n",
    "        #ngrams preceding Wn-1\n",
    "        \n",
    "        #for quadgram\n",
    "        quad_first_dict, quad_sec_dict = createKNDict(quad_dict, 4)\n",
    "        \n",
    "        #for trigram\n",
    "        tri_first_dict, tri_sec_dict = createKNDict(tri_dict, 3)\n",
    "        \n",
    "        #for bigram\n",
    "        bi_first_dict, bi_sec_dict = createKNDict(bi_dict, 2)\n",
    "       \n",
    "        #now find the probability for the trigram sentences\n",
    "        for quad in quad_dict:\n",
    "            quad_token = quad.split()\n",
    "            #W1,W2,W3\n",
    "            tri_sen = ' '.join(quad_token[:3])\n",
    "            \n",
    "            #the forumula has been divied into parts for easier understanding\n",
    "            quad_prob1 = max( quad_dict[quad] - d, 0) / tri_dict[tri_sen]\n",
    "            quad_prob2 = d/tri_dict[tri_sen] * ( quad_sec_dict[tri_sen] )\n",
    "            \n",
    "            tri_prob1 = max( tri_first_dict[quad_token[-1]] - d, 0) / len(tri_dict)\n",
    "            tri_prob2 = ( d/len(tri_dict) )* ( tri_sec_dict[' '.join(quad_token[1:3])] )\n",
    "            \n",
    "            bi_prob1 = max( bi_first_dict[quad_token[-1]] - d, 0) / len(bi_dict)\n",
    "            bi_prob2 = ( d/len(bi_dict) ) * ( bi_sec_dict[quad_token[-2]])\n",
    "            uni_prob = bi_first_dict[quad_token[-1]] / len(bi_dict)\n",
    "            \n",
    "            prob = quad_prob1 + quad_prob2*( tri_prob1 + tri_prob2*( bi_prob1 + bi_prob2* uni_prob ) )\n",
    "           \n",
    "            if tri_sen not in prob_dict:\n",
    "                prob_dict[tri_sen] = []\n",
    "                prob_dict[tri_sen].append([prob,quad_token[-1]])\n",
    "            else:\n",
    "                prob_dict[tri_sen].append([prob,quad_token[-1]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def computeKnesserNeyProb1(vocab_dict, bi_dict, tri_dict, quad_dict, prob_dict ):\n",
    "        d = 0.75\n",
    "        \n",
    "        #first create the dict for storing the count of Wn-1 followed by Wn and for\n",
    "        #ngrams preceding Wn-1\n",
    "        \n",
    "        #for quadgram\n",
    "        quad_first_dict, quad_sec_dict = createKNDict(quad_dict, 4)\n",
    "        \n",
    "        #for trigram\n",
    "        tri_first_dict, tri_sec_dict = createKNDict(tri_dict, 3)\n",
    "        \n",
    "        #for bigram\n",
    "        bi_first_dict, bi_sec_dict = createKNDict(bi_dict, 2)\n",
    "        \n",
    "        #now find the probability for the trigram sentences\n",
    "        for tri in tri_dict:\n",
    "            max_prob1 = 0.0\n",
    "            max_prob2 = 0.0\n",
    "            curr_prob = 0.0\n",
    "            word1 = ''\n",
    "            word2 = ''\n",
    "            for word in vocab_dict:\n",
    "                quad = tri + ' ' + word\n",
    "                quad_token = quad.split()\n",
    "                #W1,W2,W3\n",
    "                tri_sen = ' '.join(quad_token[:3])\n",
    "\n",
    "                #the forumula has been divied into parts for easier understanding\n",
    "                quad_prob1 = max( quad_dict[quad] - d, 0) / tri_dict[tri_sen]\n",
    "                quad_prob2 = d/tri_dict[tri_sen] * ( quad_sec_dict[tri_sen] )\n",
    "\n",
    "                tri_prob1 = max( tri_first_dict[quad_token[-1]] - d, 0) / len(tri_dict)\n",
    "                tri_prob2 = ( d/len(tri_dict) )* ( tri_sec_dict[' '.join(quad_token[1:3])] )\n",
    "\n",
    "                bi_prob1 = max( bi_first_dict[quad_token[-1]] - d, 0) / len(bi_dict)\n",
    "                bi_prob2 = ( d/len(bi_dict) ) * ( bi_sec_dict[quad_token[-2]])\n",
    "                uni_prob = bi_first_dict[quad_token[-1]] / len(bi_dict)\n",
    "\n",
    "                curr_prob = quad_prob1 + quad_prob2*( tri_prob1 + tri_prob2*( bi_prob1 + bi_prob2* uni_prob ) )\n",
    "                \n",
    "                if curr_prob > max_prob1:\n",
    "                    max_prob1 = curr_prob\n",
    "                    word1 = quad_token[-1]\n",
    "                else:\n",
    "                    if curr_prob > max_prob2:\n",
    "                        max_prob2 = curr_prob\n",
    "                        word2 = quad_token[-1]\n",
    "                    \n",
    "            \n",
    "            prob_dict[tri] = []\n",
    "            prob_dict[tri].append([max_prob1,word1])\n",
    "            prob_dict[tri].append([max_prob2,word2])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## <u>Driver Function for Testing the Language Model</u>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#return: void\n",
    "#arg:string,string,dict,dict,dict,dict,dict\n",
    "#Used for testing the Language Model\n",
    "def trainCorpus(train_file,test_file,bi_dict,tri_dict,quad_dict,vocab_dict,prob_dict):\n",
    "      \n",
    "    score = 0\n",
    "    #load the training corpus for the dataset\n",
    "    #token_len = loadCorpus1(bi_dict,tri_dict,quad_dict,vocab_dict)\n",
    "    token_len = loadCorpus(train_file, bi_dict, tri_dict, quad_dict, vocab_dict)\n",
    "    print(\"---Processing Time for Corpus Loading: %s seconds ---\" % (time.time() - start_time))\n",
    "\n",
    "    start_time1 = time.time()\n",
    "    #compute the Knesser Ney probabilities\n",
    "    computeKnesserNeyProb1(vocab_dict, bi_dict, tri_dict, quad_dict, prob_dict )\n",
    "    #sort the probable words by their probability\n",
    "    sortProbWordDict(prob_dict)\n",
    "    gc.collect()\n",
    "    print(\"---Preprocessing Time for Creating Probable Word Dict: %s seconds ---\" % (time.time() - start_time1))\n",
    "    \n",
    "\n",
    "### TESTING WITH TEST CORPUS\n",
    "    test_data = ''\n",
    "    #Now load the test corpus\n",
    "    #!!!!!!!!!!!!!ONLY FOR THIS CORPUS!!!!!!!!    \n",
    "    with open('test_corpus.txt','r') as file :\n",
    "        for line in file:\n",
    "            line_token = line.split()\n",
    "            test_data = test_data + ' '+ ' '.join(line_token[1:])\n",
    "           \n",
    "    ### TESTING WITH TEST CORPUS\n",
    "    test_data = ''\n",
    "    #Now load the test corpus\n",
    "    with open('test_corpus.txt','r') as file :\n",
    "        test_data = file.read()\n",
    "        \n",
    "    #remove punctuations from the test data\n",
    "    test_data = removePunctuations(test_data)\n",
    "    test_token = test_data.split()\n",
    "\n",
    "    #split the test data into 4 words list\n",
    "    test_token = test_data.split()\n",
    "    test_quadgrams = list(ngrams(test_token,4))\n",
    "\n",
    "    #print(len(test_token))\n",
    "    start_time1 = time.time()\n",
    "    score = computeTestScore(test_quadgrams,tri_dict,quad_dict,vocab_dict,prob_dict)\n",
    "    print('Score:',score)\n",
    "    print(\"---Processing Time for computing score: %s seconds ---\" % (time.time() - start_time1))\n",
    "\n",
    "    start_time2 = time.time()\n",
    "    perplexity = computePerplexity(test_token,token_len,tri_dict,quad_dict,vocab_dict,prob_dict)\n",
    "    print('Perplexity:',perplexity)\n",
    "    print(\"---Processing Time for computing Perplexity: %s seconds ---\" % (time.time() - start_time2))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## <u>main function</u>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "ename": "IndentationError",
     "evalue": "expected an indented block (<ipython-input-13-7c9be7dbc273>, line 3)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;36m  File \u001b[0;32m\"<ipython-input-13-7c9be7dbc273>\"\u001b[0;36m, line \u001b[0;32m3\u001b[0m\n\u001b[0;31m    vocab_dict = defaultdict(int)          #for storing the different words with their frequencies\u001b[0m\n\u001b[0m             ^\u001b[0m\n\u001b[0;31mIndentationError\u001b[0m\u001b[0;31m:\u001b[0m expected an indented block\n"
     ]
    }
   ],
   "source": [
    "def main():\n",
    "#variable declaration\n",
    "vocab_dict = defaultdict(int)          #for storing the different words with their frequencies    \n",
    "bi_dict = defaultdict(int)             #for keeping count of sentences of two words\n",
    "tri_dict = defaultdict(int)            #for keeping count of sentences of three words\n",
    "quad_dict = defaultdict(int)           #for keeping count of sentences of four words\n",
    "prob_dict = defaultdict(list)         #for storing probability of probable words for prediction\n",
    "\n",
    "train_file = 'corpusfile.txt'\n",
    "token_len = loadCorpus(train_file, bi_dict, tri_dict, quad_dict, vocab_dict)\n",
    "\n",
    "#compute the Knesser Ney probabilities for bigrams,trigrams and unigrams\n",
    "computeKnesserNeyProb(vocab_dict, bi_dict, tri_dict, quad_dict, prob_dict )\n",
    "\n",
    "#sort the probable words by their probability\n",
    "sortProbWordDict(prob_dict)\n",
    "\n",
    "#take user input \n",
    "input_sen = takeInput()\n",
    "\n",
    "### PREDICTION\n",
    "prediction = doPrediction(input_sen,prob_dict)\n",
    "print('Word Prediction:',prediction)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if __name__ == '__main__':\n",
    "    main()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## <i><u>For Debugging Purpose Only</u></i>\n",
    "<i>Ignore running the cells below if not debugging</i>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---Preprocessing Time for Corpus loading: 2.9048519134521484 seconds ---\n"
     ]
    }
   ],
   "source": [
    "#variable declaration\n",
    "vocab_dict = defaultdict(int)          #for storing the different words with their frequencies    \n",
    "bi_dict = defaultdict(int)             #for keeping count of sentences of two words\n",
    "tri_dict = defaultdict(int)            #for keeping count of sentences of three words\n",
    "quad_dict = defaultdict(int)           #for keeping count of sentences of four words\n",
    "prob_dict = defaultdict(list)         #for storing probability of probable words for prediction\n",
    "\n",
    "#load the corpus for the dataset\n",
    "#loadCorpus('corpusfile.txt',bi_dict,tri_dict,quad_dict,vocab_dict)\n",
    "print(\"---Preprocessing Time for Corpus loading: %s seconds ---\" % (time.time() - start_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### For Testing the Language Model\n",
    "Calculates % Accuracy and Perplexity<br>\n",
    "NOTE : If this is run then no need to run the cells following it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---Processing Time for Corpus Loading: 16.079257488250732 seconds ---\n"
     ]
    }
   ],
   "source": [
    "train_file = 'training_corpus.txt'\n",
    "test_file = 'test_corpus.txt'\n",
    "#load the corpus for the dataset\n",
    "token_len = trainCorpus(train_file,test_file,bi_dict,tri_dict,quad_dict,vocab_dict,prob_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_file = 'corpusfile.txt'\n",
    "token_len = loadCorpus(train_file, bi_dict, tri_dict, quad_dict, vocab_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#compute the Knesser Ney probabilities for bigrams,trigrams and unigrams\n",
    "computeKnesserNeyProb(vocab_dict, bi_dict, tri_dict, quad_dict, prob_dict )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#sort the probable words by their probability\n",
    "sortProbWordDict(prob_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#take user input \n",
    "input_sen = takeInput()\n",
    "\n",
    "### PREDICTION\n",
    "start_time2 = time.time()\n",
    "prediction = doPrediction(input_sen,prob_dict)\n",
    "print('Word Prediction:',prediction)\n",
    "print(\"---Time for Prediction Operation: %s seconds ---\" % (time.time() - start_time2))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#FOR DEBUGGING ONLY\n",
    "writeProbDicts(prob_dict)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}