SVMSemanticClassifier.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import collections\n",
    "import nltk.classify.util, nltk.metrics\n",
    "from nltk.classify import SklearnClassifier\n",
    "from sklearn.naive_bayes import BernoulliNB\n",
    "from sklearn.svm import SVC\n",
    "from nltk.corpus import movie_reviews\n",
    "from nltk.metrics import scores\n",
    "from nltk import precision\n",
    "import itertools\n",
    "from nltk.collocations import BigramCollocationFinder\n",
    "from nltk.metrics import BigramAssocMeasures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def evaluate_classifier(featx,collocationFunc):\n",
    "    #negFiles = movie_reviews.fileids('neg')\n",
    "    #posFiles = movie_reviews.fileids('pos')\n",
    "    #negWordsList=[movie_reviews.words(fileids=[f]) for f in negFiles]\n",
    "    #posWordsList=[movie_reviews.words(fileids=[f]) for f in posFiles]\n",
    "    #negfeats = [(featx(negWords), 'neg') for negWords in negWordsList]\n",
    "    #posfeats = [(featx(posWords), 'pos') for posWords in posWordsList]\n",
    "\n",
    "    negids = movie_reviews.fileids('neg')\n",
    "    posids = movie_reviews.fileids('pos')\n",
    " \n",
    "    negfeats = [(featx(movie_reviews.words(fileids=[f]),collocationFunc), 'neg') for f in negids]\n",
    "    posfeats = [(featx(movie_reviews.words(fileids=[f]),collocationFunc), 'pos') for f in posids]\n",
    "\n",
    "    #lenNegFeats=min(len(negfeats),24)\n",
    "    #lenPosFeats=min(len(posfeats),24)\n",
    "    lenNegFeats=len(negfeats)\n",
    "    lenPosFeats=len(posfeats)\n",
    "    negcutoff = int(lenNegFeats*3/4)\n",
    "    poscutoff = int(lenPosFeats*3/4)\n",
    " \n",
    "    trainfeats = negfeats[:negcutoff] + posfeats[:poscutoff]\n",
    "    testfeats = negfeats[negcutoff:lenNegFeats] + posfeats[poscutoff:lenPosFeats]\n",
    " \n",
    "    #classifier = MaxentClassifier.train(trainfeats)\n",
    "    classifier = SklearnClassifier(BernoulliNB()).train(trainfeats)\n",
    "    refsets = collections.defaultdict(set)\n",
    "    testsets = collections.defaultdict(set)\n",
    " \n",
    "    for i, (feats, label) in enumerate(testfeats):\n",
    "            refsets[label].add(i)\n",
    "            observed = classifier.classify(feats)\n",
    "            testsets[observed].add(i)\n",
    "    evaluationMetrics={}\n",
    "    print(classifier)\n",
    "    evaluationMetrics['accuracy']=nltk.classify.util.accuracy(classifier, testfeats)\n",
    "    evaluationMetrics['posPrec']=nltk.precision(refsets['pos'], testsets['pos'])\n",
    "    evaluationMetrics['posRecall']=nltk.recall(refsets['pos'], testsets['pos'])\n",
    "    evaluationMetrics['posF_Score']=nltk.f_measure(refsets['pos'], testsets['pos'])\n",
    "    evaluationMetrics['negPrec']=nltk.precision(refsets['neg'], testsets['neg'])\n",
    "    evaluationMetrics['negRecall']=nltk.recall(refsets['neg'], testsets['neg'])\n",
    "    evaluationMetrics['negF_Score']=nltk.f_measure(refsets['neg'], testsets['neg'])\n",
    "    return evaluationMetrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<SklearnClassifier(BernoulliNB(alpha=1.0, binarize=0.0, class_prior=None, fit_prior=True))>\n"
     ]
    }
   ],
   "source": [
    "from nltk.corpus import stopwords\n",
    "stopset = set(stopwords.words('english'))\n",
    "evaluations=[] \n",
    "def stopword_filtered_word_feats(words,collocator):\n",
    "    return dict([(word, True) for word in words if word not in stopset])\n",
    "evaluations.append(evaluate_classifier(stopword_filtered_word_feats,None)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'accuracy': 0.808,\n",
       "  'negF_Score': 0.8285714285714286,\n",
       "  'negPrec': 0.7483870967741936,\n",
       "  'negRecall': 0.928,\n",
       "  'posF_Score': 0.7818181818181817,\n",
       "  'posPrec': 0.9052631578947369,\n",
       "  'posRecall': 0.688}]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "unorderable types: str() < tuple()",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-13-2b50d840c4d2>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      9\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mdict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mngram\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mngram\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mitertools\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mchain\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mwords\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbigrams\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     10\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 11\u001b[1;33m \u001b[0mevaluations\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mevaluate_classifier\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbigram_word_feats\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mBigramAssocMeasures\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mchi_sq\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;31m#Works best for this Data\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     12\u001b[0m \u001b[0mevaluations\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mevaluate_classifier\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbigram_word_feats\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mBigramAssocMeasures\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mjaccard\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     13\u001b[0m \u001b[0mevaluations\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mevaluate_classifier\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbigram_word_feats\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mBigramAssocMeasures\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlikelihood_ratio\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-11-87ad90773217>\u001b[0m in \u001b[0;36mevaluate_classifier\u001b[1;34m(featx, collocationFunc)\u001b[0m\n\u001b[0;32m     24\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     25\u001b[0m     \u001b[1;31m#classifier = MaxentClassifier.train(trainfeats)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 26\u001b[1;33m     \u001b[0mclassifier\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mSklearnClassifier\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mBernoulliNB\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrainfeats\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     27\u001b[0m     \u001b[0mrefsets\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcollections\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdefaultdict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mset\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     28\u001b[0m     \u001b[0mtestsets\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcollections\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdefaultdict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mset\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/alok/anaconda3/lib/python3.5/site-packages/nltk/classify/scikitlearn.py\u001b[0m in \u001b[0;36mtrain\u001b[1;34m(self, labeled_featuresets)\u001b[0m\n\u001b[0;32m    113\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    114\u001b[0m         \u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcompat\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mizip\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0mlabeled_featuresets\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 115\u001b[1;33m         \u001b[0mX\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_vectorizer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit_transform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    116\u001b[0m         \u001b[0my\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_encoder\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit_transform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    117\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_clf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/alok/anaconda3/lib/python3.5/site-packages/sklearn/feature_extraction/dict_vectorizer.py\u001b[0m in \u001b[0;36mfit_transform\u001b[1;34m(self, X, y)\u001b[0m\n\u001b[0;32m    224\u001b[0m             \u001b[0mFeature\u001b[0m \u001b[0mvectors\u001b[0m\u001b[1;33m;\u001b[0m \u001b[0malways\u001b[0m \u001b[1;36m2\u001b[0m\u001b[1;33m-\u001b[0m\u001b[0md\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    225\u001b[0m         \"\"\"\n\u001b[1;32m--> 226\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_transform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfitting\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    227\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    228\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0minverse_transform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdict_type\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdict\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/alok/anaconda3/lib/python3.5/site-packages/sklearn/feature_extraction/dict_vectorizer.py\u001b[0m in \u001b[0;36m_transform\u001b[1;34m(self, X, fitting)\u001b[0m\n\u001b[0;32m    188\u001b[0m         \u001b[1;31m# Sort everything if asked\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    189\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mfitting\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msort\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 190\u001b[1;33m             \u001b[0mfeature_names\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msort\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    191\u001b[0m             \u001b[0mmap_index\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mempty\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfeature_names\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mint32\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    192\u001b[0m             \u001b[1;32mfor\u001b[0m \u001b[0mnew_val\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mf\u001b[0m \u001b[1;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfeature_names\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mTypeError\u001b[0m: unorderable types: str() < tuple()"
     ]
    }
   ],
   "source": [
    "#Bigram Collocations- Handle Cases like “not good”, here B-O-W Approach will Fail\n",
    "import itertools\n",
    "from nltk.collocations import BigramCollocationFinder\n",
    "from nltk.metrics import BigramAssocMeasures\n",
    " \n",
    "def bigram_word_feats(words, score_fn, n=200):\n",
    "    bigram_finder = BigramCollocationFinder.from_words(words)\n",
    "    bigrams = bigram_finder.nbest(score_fn, n)\n",
    "    return dict([(ngram, True) for ngram in itertools.chain(words, bigrams)])\n",
    " \n",
    "evaluations.append(evaluate_classifier(bigram_word_feats,BigramAssocMeasures.chi_sq))#Works best for this Data\n",
    "evaluations.append(evaluate_classifier(bigram_word_feats,BigramAssocMeasures.jaccard))\n",
    "evaluations.append(evaluate_classifier(bigram_word_feats,BigramAssocMeasures.likelihood_ratio))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from nltk.collocations import *\n",
    "from nltk.probability import FreqDist\n",
    "from nltk.probability import ConditionalFreqDist\n",
    "word_fd = FreqDist()\n",
    "label_word_fd = ConditionalFreqDist()\n",
    "\n",
    "testNegWords = movie_reviews.words(categories=['pos'])\n",
    "testPosWords = movie_reviews.words(categories=['neg'])\n",
    "\n",
    "for word in testNegWords:\n",
    "    word_fd[word.lower()]+=1\n",
    "    label_word_fd['neg'][word.lower()]+=1\n",
    "for word in testPosWords:\n",
    "    word_fd[word.lower()]+=1\n",
    "    label_word_fd['pos'][word.lower()]+=1\n",
    "print(word_fd.N(),word_fd.B(),word_fd.most_common(20))\n",
    "print(label_word_fd.N(),label_word_fd.conditions(),label_word_fd.items())\n",
    "print(label_word_fd['pos'].N(),label_word_fd['neg'].N())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# n_ii = label_word_fd[label][word]\n",
    "# n_ix = word_fd[word]\n",
    "# n_xi = label_word_fd[label].N()\n",
    "# n_xx = label_word_fd.N()\n",
    "#         w1    ~w1\n",
    "#      ------ ------\n",
    "#  w2 | n_ii | n_oi | = n_xi\n",
    "#      ------ ------\n",
    "# ~w2 | n_io | n_oo |\n",
    "#     ------ ------\n",
    "#      =n_ix         TOTAL = n_xx\n",
    "# A number of measures are available to score collocations or other associations. The arguments to measure \n",
    "# functions are marginals of a contingency table, in the bigram case (n_ii, (n_ix, n_xi), n_xx):\n",
    "# n_ii = label_word_fd[label][word]\n",
    "# n_ix = word_fd[word]\n",
    "# n_xi = label_word_fd[label].N()\n",
    "# n_xx = label_word_fd.N()\n",
    "# Chi-Sq Contingency Table : Relating Word w1 with \"pos\" classification \n",
    "#         w1    ~w1\n",
    "#      ------ ------\n",
    "# +ve | n_ii | n_oi | = n_xi\n",
    "#      ------ ------\n",
    "# -ve | n_io | n_oo |\n",
    "#     ------ ------\n",
    "#      =n_ix         TOTAL = n_xx\n",
    "# n_ix : Total Freq of word w1, n_xi: pos_word_count \n",
    "pos_word_count = label_word_fd['pos'].N()\n",
    "neg_word_count = label_word_fd['neg'].N()\n",
    "total_word_count = pos_word_count + neg_word_count\n",
    " \n",
    "word_scores = {}\n",
    "\n",
    "#print(word_fd.items())\n",
    "for word, freq in word_fd.items():\n",
    "    pos_score = BigramAssocMeasures.chi_sq(label_word_fd['pos'][word],(freq, pos_word_count), total_word_count)\n",
    "    neg_score = BigramAssocMeasures.chi_sq(label_word_fd['neg'][word],(freq, neg_word_count), total_word_count)\n",
    "    word_scores[word] = pos_score + neg_score\n",
    "import operator\n",
    "best1 = sorted(word_scores.items(), key=operator.itemgetter(1), reverse=True)[:10000]\n",
    "bestwords = set([w for w, s in best1])\n",
    " \n",
    "def best_word_feats(words,biGramMeasure):\n",
    "    return dict([(word, True) for word in words if word in bestwords])\n",
    " \n",
    "evaluations.append(evaluate_classifier(best_word_feats,BigramAssocMeasures.chi_sq))\n",
    " \n",
    "def best_bigram_word_feats(words, score_fn=BigramAssocMeasures.chi_sq, n=200):\n",
    "    bigram_finder = BigramCollocationFinder.from_words(words)\n",
    "    bigrams = bigram_finder.nbest(score_fn, n)\n",
    "    d = dict([(bigram, True) for bigram in bigrams])\n",
    "    d.update(best_word_feats(words,score_fn))\n",
    "    return d\n",
    "evaluations.append(evaluate_classifier(best_bigram_word_feats,BigramAssocMeasures.chi_sq))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for modelEvalMetrics in evaluations:\n",
    "    print(modelEvalMetrics)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}