pv_email.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test Project: Import emails exported from outlook via csv into DOTCE, classify"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 1: Go to correct directory and activate the DOTCE virtual env"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "c:\\Users\\pviechnicki\\Desktop\\pviechnicki_home\\sandbox\\state\\dotce\n",
      "# conda environments:\n",
      "#\n",
      "dotce                    C:\\Users\\pviechnicki\\AppData\\Local\\Continuum\\Anaconda3\\envs\\dotce\n",
      "root                  *  C:\\Users\\pviechnicki\\AppData\\Local\\Continuum\\Anaconda3\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%cd c:\\Users\\pviechnicki\\Desktop\\pviechnicki_home\\sandbox\\state\\dotce\n",
    "%pwd\n",
    "! activate dotce\n",
    "! conda info --envs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 2: Load various machine learning libraries to extract features from text document corpus and build classification models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\pviechnicki\\AppData\\Local\\Continuum\\Anaconda3\\envs\\dotce\\lib\\site-packages\\sklearn\\cross_validation.py:44: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import numpy as np\n",
    "import sys\n",
    "sys.path.append('C:\\\\Users\\\\pviechnicki\\\\AppData\\\\Local\\\\Continuum\\\\Anaconda3\\\\envs\\\\dotce\\\\lib\\\\site-packages\\\\pyLDAvis')\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from collections import Counter\n",
    "import string\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem.snowball import SnowballStemmer\n",
    "from sklearn.cross_validation import train_test_split\n",
    "from sklearn.linear_model import LogisticRegression as LR\n",
    "from sklearn.naive_bayes import MultinomialNB"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 3: Read csv file of emails into df and split into training and test sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original dataframe contains 4629 messages\n",
      "Non-empty datafram contains 925 messages\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#Make sure we're in the right directory\n",
    "os.getcwd()\n",
    "email_df = pd.read_csv('C:\\\\Users\\\\pviechnicki\\\\Desktop\\\\pviechnicki_home\\\\sandbox\\\\state\\\\data\\\\pv_email\\\\combined_emails.csv', sep='|')\n",
    "#Add rowid\n",
    "email_df['rownum'] = range(0, len(email_df))\n",
    "email_df.groupby('cat').count()\n",
    "# Filter out empty rows\n",
    "non_empty_df = email_df[email_df['body'].isnull() == False].sample(frac=.2)\n",
    "#sample method chooses a random sample of the origina frame\n",
    "#https://stackoverflow.com/questions/29576430/shuffle-dataframe-rows\n",
    "print(\"Original dataframe contains {} messages\\nNon-empty datafram contains {} messages\\n\".format(\n",
    "len(email_df), len(non_empty_df)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False    654\n",
       "True     271\n",
       "Name: about_fraud, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Add new column to dataframe with True if cat == fraud_waste_abuse\n",
    "#Edit the column names and truth conditions to match your data\n",
    "non_empty_df['about_fraud'] = (non_empty_df['cat'] == 'fraud_waste_abuse')\n",
    "#Create a vector of class labels\n",
    "class_labels = non_empty_df['about_fraud']\n",
    "#use value_counts() method of series\n",
    "class_labels.value_counts()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Create a training set and test set, 80% 20%\n",
    "train_df, test_df = train_test_split(non_empty_df, train_size = 0.8, random_state=44)\n",
    "class_labels_training = list(train_df['about_fraud'])\n",
    "class_labels_test = list(test_df['about_fraud'])\n",
    "value_counts = nltk.FreqDist(class_labels_training)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 4: Let's make sure we can tokenize these properly and remove stop words\n",
    "<p>from <a href=\"http://www.cs.duke.edu/courses/spring14/compsci290/assignments/lab02.html\">CS Duke.edu</a></p>\n",
    "<p>Make sure you've installed the english punctuation and stop words list \n",
    "following <a href=\"http://www.nltk.org/data.html\">these instructions.</a>\n",
    "</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('us', 4), ('cyber', 3), ('articl', 3), ('egger', 3), ('william', 3), ('arlington', 3), ('sent', 3), ('march', 2), ('2016', 2), ('weggersdeloittecom', 2)]\n"
     ]
    }
   ],
   "source": [
    "#Instantiate a stemmer and a tokenizer to preprocess the email text\n",
    "\n",
    "snowballStemmer = SnowballStemmer(\"english\", ignore_stopwords=True)\n",
    "def preprocess(text):\n",
    "    no_punctuation_text = ''\n",
    "    if (type(text)== str):\n",
    "        lower_text = text.lower()\n",
    "        no_punctuation_text = lower_text.translate({ord(c):'' for c in string.punctuation})\n",
    "    return no_punctuation_text\n",
    "\n",
    "def myTokenize(text):\n",
    "    global snowballStemmer\n",
    "    tokens = []\n",
    "    cleaned = preprocess(text)\n",
    "    tokens = nltk.word_tokenize(cleaned)\n",
    "    filtered = [w for w in tokens if not w in stopwords.words('english')]\n",
    "    stemmed = [w for w in map(snowballStemmer.stem, filtered)]\n",
    "    return stemmed\n",
    "    \n",
    "tokens = myTokenize(train_df['body'].iloc[0])\n",
    "count = Counter(tokens)\n",
    "print(count.most_common(10))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 5: convert train and test dfs to term X document representation matrices (_X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "##Instantiate a TFidf vectorizer\n",
    "vectorizer = TfidfVectorizer(sublinear_tf=True, encoding='utf-8', \n",
    "                             max_df=0.5, tokenizer=myTokenize)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#This step can take a while\n",
    "train_X = vectorizer.fit_transform(train_df['body'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "##Save feature names in a separate list\n",
    "feature_names = vectorizer.get_feature_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#From http://fastml.com/classifying-text-with-bag-of-words-a-tutorial/\n",
    "#Create another matrix of tfidf scores for the documents in the test set\n",
    "test_X = vectorizer.transform(test_df['body'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 5: Instantiate and Train a Logistic Regression Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Create a logistic regression model\n",
    "#From same tutorial https://github.com/zygmuntz/classifying-text/blob/master/bow_predict.py\n",
    "model_lr = LR()\n",
    "model_lr.fit(train_X, class_labels_training)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 5B: Try a Naive Bayes Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MultinomialNB(alpha=0.05, class_prior=None, fit_prior=True)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Try a naive bayes classifier instead\n",
    "model_nb = MultinomialNB(alpha=0.05)\n",
    "model_nb.fit(train_X, class_labels_training)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 6: Test classifier on test_X matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Predict probability of class membership\n",
    "p = model_lr.predict_proba( test_X )[:,1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test NB Classifier results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.98378378378378384"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p2 = model_nb.predict( test_X )\n",
    "model_nb.score( test_X, class_labels_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(True, True),\n",
       " (False, False),\n",
       " (True, True),\n",
       " (False, False),\n",
       " (False, False),\n",
       " (False, False),\n",
       " (True, True),\n",
       " (False, False),\n",
       " (False, False),\n",
       " (True, True)]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results = [(class_labels_test[i], p2[i]) for i in range(0,len(p2))]\n",
    "len(results)\n",
    "results[0:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 7: Evaluate Results\n",
    "<p>Nice summary of different formulas for accuracy, precision, recall, etc \n",
    "<a href=\"http://www.damienfrancois.be/blog/files/modelperfcheatsheet.pdf\">here</a></p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Write out results\n",
    "output_df = pd.DataFrame( data = {'rownum': test_df['rownum'], \n",
    "                                  'about_fraud': test_df['about_fraud'], \n",
    "                                  'logistic': p })\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Output dataframe length: 185\n",
      "\n",
      "DF columns:about_fraud, logistic, rownum, prediction\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>about_fraud</th>\n",
       "      <th>logistic</th>\n",
       "      <th>rownum</th>\n",
       "      <th>prediction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1605</th>\n",
       "      <td>False</td>\n",
       "      <td>0.060560</td>\n",
       "      <td>1605</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1707</th>\n",
       "      <td>False</td>\n",
       "      <td>0.153765</td>\n",
       "      <td>1707</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>745</th>\n",
       "      <td>False</td>\n",
       "      <td>0.154392</td>\n",
       "      <td>745</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2263</th>\n",
       "      <td>False</td>\n",
       "      <td>0.179762</td>\n",
       "      <td>2263</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3977</th>\n",
       "      <td>True</td>\n",
       "      <td>0.706092</td>\n",
       "      <td>3977</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1396</th>\n",
       "      <td>False</td>\n",
       "      <td>0.111080</td>\n",
       "      <td>1396</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>False</td>\n",
       "      <td>0.353156</td>\n",
       "      <td>60</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4283</th>\n",
       "      <td>True</td>\n",
       "      <td>0.398459</td>\n",
       "      <td>4283</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3621</th>\n",
       "      <td>True</td>\n",
       "      <td>0.895062</td>\n",
       "      <td>3621</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3086</th>\n",
       "      <td>False</td>\n",
       "      <td>0.121713</td>\n",
       "      <td>3086</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      about_fraud  logistic  rownum  prediction\n",
       "1605        False  0.060560    1605       False\n",
       "1707        False  0.153765    1707       False\n",
       "745         False  0.154392     745       False\n",
       "2263        False  0.179762    2263       False\n",
       "3977         True  0.706092    3977        True\n",
       "1396        False  0.111080    1396       False\n",
       "60          False  0.353156      60        True\n",
       "4283         True  0.398459    4283        True\n",
       "3621         True  0.895062    3621        True\n",
       "3086        False  0.121713    3086       False"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Output dataframe length: {}\\n\".format(len(output_df)))\n",
    "THRESHOLD = .23\n",
    "output_df['prediction'] = (output_df['logistic'] >= THRESHOLD)\n",
    "print(\"DF columns:{}\\n\".format(\", \".join(output_df.columns)))\n",
    "\n",
    "##If 'prediction' matches 'about_fraud' then the classifier got it right\n",
    "output_df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results\n",
      "True Positives\tTrue_Negatives\tFalse_Positives\\False_Negatives\n",
      "\n",
      "58\t99\t28\t0\n",
      "Classifier Accuracy: 0.8486486486486486\n",
      "\n",
      "Classifier Error Rate: 0.15135135135135136\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#Now let's see how well this model did, true positives, false positives, etc\n",
    "def accuracy(tp, tn, fp, fn):\n",
    "    return ((tp + tn)/(tp + tn + fp + fn))\n",
    "\n",
    "def error_rate(tp, tn, fp, fn):\n",
    "    return ((fp + fn)/ (tp + tn + fp + fn))\n",
    "\n",
    "true_positives = len(output_df.loc[(output_df['about_fraud'] == True) & (output_df['prediction'] == True)])\n",
    "false_positives = len(output_df.loc[(output_df['about_fraud'] == False) & (output_df['prediction'] == True)])\n",
    "true_negatives = len(output_df.loc[(output_df['about_fraud'] == False) & (output_df['prediction'] == False)])\n",
    "false_negatives = len(output_df.loc[(output_df['about_fraud'] == True) & (output_df['prediction'] == False)])\n",
    "print(\"Results\\nTrue Positives\\tTrue_Negatives\\tFalse_Positives\\False_Negatives\\n\")\n",
    "print(\"\\t\".join(map(str, [true_positives, true_negatives, false_positives, false_negatives])))\n",
    "\n",
    "print(\"Classifier Accuracy: {}\\n\".format(accuracy(true_positives, true_negatives, \n",
    "                                                  false_positives, false_negatives)))\n",
    "print(\"Classifier Error Rate: {}\\n\".format(error_rate(true_positives, true_negatives,\n",
    "                                                      false_positives, false_negatives)))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluate NB using metrics module"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "      False       0.98      0.99      0.99       132\n",
      "       True       0.98      0.96      0.97        53\n",
      "\n",
      "avg / total       0.98      0.98      0.98       185\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn import metrics\n",
    "print(metrics.classification_report(class_labels_test, p2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Which features are most informative?\n",
    "<p>Stolen/lifted from <a href=\"https://stackoverflow.com/questions/11116697/how-to-get-most-informative-features-for-scikit-learn-classifiers\">stack overflow</a></p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\t-10.9992\t\u0002ià\u000e           \t\t-5.5931\tfraud          \n",
      "\t-10.9992\t\u00049            \t\t-5.8321\twast           \n",
      "\t-10.9992\t\u0016\u0016nyth         \t\t-5.9793\t2016           \n",
      "\t-10.9992\t00000          \t\t-6.0849\tpulkit         \n",
      "\t-10.9992\t000000000000000\t\t-6.1279\twilliam        \n",
      "\t-10.9992\t0000000000000000\t\t-6.2162\tkapoor         \n",
      "\t-10.9992\t001            \t\t-6.2896\tegger          \n",
      "\t-10.9992\t002doc         \t\t-6.2910\tbrien          \n",
      "\t-10.9992\t002pptx        \t\t-6.2940\tprogram        \n",
      "\t-10.9992\t00368805037logbinglog\t\t-6.3046\thttpgovernment2020dupresscom\n",
      "\t-10.9992\t0059jerjohnstondeloittecom\t\t-6.3225\tjake           \n",
      "\t-10.9992\t0065           \t\t-6.3451\tabus           \n",
      "\t-10.9992\t0099600070300024709643002367066790713830001socialindexindex\t\t-6.3562\tweggersdeloittecom\n",
      "\t-10.9992\t011425003770054180325700849101231creativeindexindex\t\t-6.3704\tstudi          \n",
      "\t-10.9992\t0120wwwdeloittecom\t\t-6.3994\tsubject        \n",
      "\t-10.9992\t0137           \t\t-6.4298\tmailtoweggersdeloittecom\n",
      "\t-10.9992\t0137krkauldeloittecom\t\t-6.4402\tversion        \n",
      "\t-10.9992\t0199390000300273806196pmindexindex\t\t-6.4418\tdan            \n",
      "\t-10.9992\t02116          \t\t-6.4421\tmumbai         \n",
      "\t-10.9992\t02142          \t\t-6.4431\thttpwwwsolutionrevolutionbookcom\n",
      "\t-10.9992\t0225           \t\t-6.4451\terror          \n",
      "\t-10.9992\t030            \t\t-6.4604\tpublic         \n",
      "\t-10.9992\t032            \t\t-6.4663\tpviechnickideloittecom\n",
      "\t-10.9992\t0368586633     \t\t-6.4987\tnew            \n",
      "\t-10.9992\t0375apansedeloittecom\t\t-6.5082\tamto           \n"
     ]
    }
   ],
   "source": [
    "#What are the most informative features in this test?\n",
    "def show_most_informative_features(vectorizer, clf, n=20):\n",
    "    feature_names = vectorizer.get_feature_names()\n",
    "    coefs_with_fns = sorted(zip(clf.coef_[0], feature_names))\n",
    "    top = zip(coefs_with_fns[:n], coefs_with_fns[:-(n + 1):-1])\n",
    "    for (coef_1, fn_1), (coef_2, fn_2) in top:\n",
    "        print (\"\\t%.4f\\t%-15s\\t\\t%.4f\\t%-15s\" % (coef_1, fn_1, coef_2, fn_2))\n",
    "\n",
    "show_most_informative_features(vectorizer, model_nb, 25)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Step 8: Optional Exploratory Data Analysis -- Check counts for specific terms\n",
    "<p><i>Not worth doing this on larger corpora -- too slow!</i></p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing email # 100\n",
      "Processing email # 200\n",
      "Processing email # 300\n",
      "Processing email # 400\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('us', 5312), ('arlington', 2462), ('peter', 1768), ('sent', 1126), ('viechnicki', 1055), ('mumbai', 800), ('1', 797), ('subject', 677), ('data', 610), ('mailtopviechnickideloittecom', 604)]\n"
     ]
    }
   ],
   "source": [
    "##Instantiate a new counter and count frequencies of the tokens\n",
    "c = Counter()\n",
    "lineNo = 0\n",
    "for text in non_empty_df['body'].tolist():\n",
    "    lineNo += 1\n",
    "    tokens = myTokenize(text)\n",
    "    c.update(tokens)\n",
    "    if (lineNo % 100 == 0):\n",
    "        sys.stderr.write(\"Processing email # {}\\n\".format(lineNo))\n",
    "print(c.most_common(10))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Step 9: More EDA -- A better way of compiling the vocabulary\n",
    "<a href=\"http://nlpforhackers.io/tf-idf/\">Source http://nlpforhackers.io/tf-idf/</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vocabulary size: 14212\n",
      "Documents Count: 740\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# build the vocabulary in one pass\n",
    "vocabulary = set()\n",
    "for text in train_df['body']:\n",
    "    words = myTokenize(text)\n",
    "    vocabulary.update(words)\n",
    " \n",
    "vocabulary = list(vocabulary)\n",
    "word_index = {w: idx for idx, w in enumerate(vocabulary)}\n",
    " \n",
    "VOCABULARY_SIZE = len(vocabulary)\n",
    "DOCUMENTS_COUNT = len(train_df.index)\n",
    " \n",
    "print(\"Vocabulary size: {}\\nDocuments Count: {}\\n\".format(\n",
    "    VOCABULARY_SIZE, DOCUMENTS_COUNT))   "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 10: Let's look more closely at the features that have high tf-idf scores\n",
    "<p>Tip of the hat to this blog: <a href=\"https://buhrmann.github.io/tfidf-analysis.html\">blog post from Thomas Buhrmann</a></p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "#Returns top n tfidf features as df, but takes dense format vector as input\n",
    "def top_tfidf_feats(row, features, top_n=25):\n",
    "    ''' Get top n tfidf values in row and return them with their corresponding feature names.'''\n",
    "    topn_ids = np.argsort(row)[::-1][:top_n]\n",
    "    top_feats = [(features[i], row[i]) for i in topn_ids]\n",
    "    df = pd.DataFrame(top_feats)\n",
    "    df.columns = ['feature', 'tfidf']\n",
    "    return df\n",
    "\n",
    "#convert single row into dense format\n",
    "def top_feats_in_doc(Xtr, features, row_id, top_n=25):\n",
    "    ''' Top tfidf features in specific document (matrix row) '''\n",
    "    row = np.squeeze(Xtr[row_id].toarray())\n",
    "    return top_tfidf_feats(row, features, top_n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature</th>\n",
       "      <th>tfidf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>mailtoweggers38gmailcom</td>\n",
       "      <td>0.475205</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>839</td>\n",
       "      <td>0.424112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>tn</td>\n",
       "      <td>0.394224</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>27</td>\n",
       "      <td>0.283355</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>weggersdeloittecomsubject</td>\n",
       "      <td>0.278087</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>octob</td>\n",
       "      <td>0.270831</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>egger</td>\n",
       "      <td>0.239569</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>william</td>\n",
       "      <td>0.237561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2015</td>\n",
       "      <td>0.233639</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>tuesday</td>\n",
       "      <td>0.153122</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     feature     tfidf\n",
       "0    mailtoweggers38gmailcom  0.475205\n",
       "1                        839  0.424112\n",
       "2                         tn  0.394224\n",
       "3                         27  0.283355\n",
       "4  weggersdeloittecomsubject  0.278087\n",
       "5                      octob  0.270831\n",
       "6                      egger  0.239569\n",
       "7                    william  0.237561\n",
       "8                       2015  0.233639\n",
       "9                    tuesday  0.153122"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#This is showing us the top ten tfidf scores for document 3\n",
    "top_feats_in_doc(tfidf_matrix, feature_names, 3, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature</th>\n",
       "      <th>tfidf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>junko</td>\n",
       "      <td>0.011140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>mission</td>\n",
       "      <td>0.008808</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>seattl</td>\n",
       "      <td>0.007536</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>httpgovernment2020dupresscom</td>\n",
       "      <td>0.007520</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>slide</td>\n",
       "      <td>0.007152</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>ai</td>\n",
       "      <td>0.007150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>httpwwwsolutionrevolutionbookcom</td>\n",
       "      <td>0.006952</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>cathryn</td>\n",
       "      <td>0.006602</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>kaji</td>\n",
       "      <td>0.006592</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>iphon</td>\n",
       "      <td>0.006308</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>fraud</td>\n",
       "      <td>0.006230</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>91</td>\n",
       "      <td>0.006227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>purva</td>\n",
       "      <td>0.006208</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>india</td>\n",
       "      <td>0.006087</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>interact</td>\n",
       "      <td>0.005927</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>617</td>\n",
       "      <td>0.005878</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>dan</td>\n",
       "      <td>0.005823</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>pankaj</td>\n",
       "      <td>0.005814</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>mahesh</td>\n",
       "      <td>0.005809</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>adam</td>\n",
       "      <td>0.005534</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>7142</td>\n",
       "      <td>0.005450</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>studi</td>\n",
       "      <td>0.005383</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>april</td>\n",
       "      <td>0.005286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>join</td>\n",
       "      <td>0.005278</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>steven</td>\n",
       "      <td>0.005260</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             feature     tfidf\n",
       "0                              junko  0.011140\n",
       "1                            mission  0.008808\n",
       "2                             seattl  0.007536\n",
       "3       httpgovernment2020dupresscom  0.007520\n",
       "4                              slide  0.007152\n",
       "5                                 ai  0.007150\n",
       "6   httpwwwsolutionrevolutionbookcom  0.006952\n",
       "7                            cathryn  0.006602\n",
       "8                               kaji  0.006592\n",
       "9                              iphon  0.006308\n",
       "10                             fraud  0.006230\n",
       "11                                91  0.006227\n",
       "12                             purva  0.006208\n",
       "13                             india  0.006087\n",
       "14                          interact  0.005927\n",
       "15                               617  0.005878\n",
       "16                               dan  0.005823\n",
       "17                            pankaj  0.005814\n",
       "18                            mahesh  0.005809\n",
       "19                              adam  0.005534\n",
       "20                              7142  0.005450\n",
       "21                             studi  0.005383\n",
       "22                             april  0.005286\n",
       "23                              join  0.005278\n",
       "24                            steven  0.005260"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Helper function to calculate top n features that are on average most important among\n",
    "#documents with grp_ids = ?\n",
    "def top_mean_feats(Xtr, features, grp_ids=None, min_tfidf=0.1, top_n=25):\n",
    "    ''' Return the top n features that on average are most important amongst documents in rows\n",
    "        indentified by indices in grp_ids. '''\n",
    "    if grp_ids:\n",
    "        D = Xtr[grp_ids].toarray()\n",
    "    else:\n",
    "        D = Xtr.toarray()\n",
    "\n",
    "    D[D < min_tfidf] = 0\n",
    "    tfidf_means = np.mean(D, axis=0)\n",
    "    return top_tfidf_feats(tfidf_means, features, top_n)\n",
    "\n",
    "#Top features for entire corpus\n",
    "top_mean_feats(train_X, feature_names, None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def top_feats_by_class(Xtr, y, features, min_tfidf=0.1, top_n=25):\n",
    "    ''' Return a list of dfs, where each df holds top_n features and their mean tfidf value\n",
    "        calculated across documents with the same class label. '''\n",
    "    dfs = []\n",
    "    ids = np.where(y.about_fraud==True)\n",
    "    feats_df = top_mean_feats(Xtr, features, ids, min_tfidf=min_tfidf, top_n=top_n)\n",
    "    feats_df.label = 'fraud_waste_abuse'\n",
    "    dfs.append(feats_df)\n",
    "    return dfs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[                         feature     tfidf\n",
       " 0                          fraud  0.032673\n",
       " 1                           wast  0.021011\n",
       " 2                           2015  0.019074\n",
       " 3                        program  0.019062\n",
       " 4                          junko  0.018963\n",
       " 5                         pulkit  0.018369\n",
       " 6                           kaji  0.016954\n",
       " 7                       interact  0.016675\n",
       " 8     pukapoordeloittecomsubject  0.015777\n",
       " 9                          brien  0.015573\n",
       " 10                          nudg  0.015497\n",
       " 11  httpgovernment2020dupresscom  0.014873\n",
       " 12                       graphic  0.014441\n",
       " 13                      strategi  0.013924\n",
       " 14                          abus  0.013748\n",
       " 15                        steven  0.013071\n",
       " 16                          séan  0.012421\n",
       " 17                         error  0.012381\n",
       " 18                          thai  0.012314\n",
       " 19                       michael  0.011989\n",
       " 20                         reduc  0.011981\n",
       " 21                            19  0.011981\n",
       " 22                          hood  0.011970\n",
       " 23                    minneapoli  0.011940\n",
       " 24             stthaideloittecom  0.011870]"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_feats_by_class(test_X, test_df, feature_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "dotce",
   "language": "python",
   "name": "dotce"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}