Preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "init_file = \"\"\"/Users/julienkervizic/Downloads/maildir/benson-r/_sent_mail/1.\"\"\"\n",
    "init_dir = \"\"\"/Users/julienkervizic/Downloads/maildir/\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import email\n",
    "import os, sys\n",
    "import codecs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "    Reads through the Enron Corpus Email Directory\n",
    "    Parse the emails and store them into a List of Dictionaries\n",
    "\"\"\"\n",
    "import sys\n",
    "reload(sys)\n",
    "sys.setdefaultencoding(\"utf-8\")\n",
    "\n",
    "emails = []\n",
    "for root, dirs, files in os.walk(init_dir, topdown=False):\n",
    "     for name in files:\n",
    "            file_path = os.path.join(root, name)\n",
    "            with codecs.open(file_path, \"r\",encoding='utf-8', errors='ignore') as f:\n",
    "                read_data = f.read()\n",
    "                import email\n",
    "                email = email.message_from_string(read_data)\n",
    "                emails.append(\n",
    "                        {\n",
    "                            'from': email['from'],\n",
    "                            'to': email['to'],\n",
    "                            'cc': email['cc'],\n",
    "                            'bcc': email['bcc'],\n",
    "                            'body': email.get_payload(decode=True)\n",
    "                        }\n",
    "                )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "    Preprocessor for Email inputs\n",
    "    1) Only considers the new inputs providing by the email corespondence\n",
    "    2) String replaces Emails, Links and Numbers\n",
    "    3) Normalize text to lowercase\n",
    "\"\"\"\n",
    "\n",
    "def PreProcessor(email_dict): \n",
    "    #only take into account the original message\n",
    "    om = \"-----Original Message-----\"\n",
    "    email_dict['body'] = email_dict['body'].split(om)[0]\n",
    "    email_dict['body'] = email_dict['body'].split(\"From:\")[0]\n",
    "    email_dict['body'] = email_dict['body'].split(\"To:\")[0]\n",
    "    \n",
    "    import re\n",
    "    # Getting rid of numbers\n",
    "    email_dict['body'] = re.sub('([0-9]+)', ' ', email_dict['body'])\n",
    "    # Replacing links\n",
    "    email_dict['body'] = re.sub('(http[^\\\"|^\\']+.)', ' ', email_dict['body'])\n",
    "    email_dict['body'] = re.sub('(www\\.+.)', '', email_dict['body'])\n",
    "    # Replacing Emails \n",
    "    email_dict['body'] = re.sub('(\\w+@\\w+\\.\\w+)', ' ', email_dict['body'])\n",
    "    # removing NONASCII\n",
    "    from unidecode import unidecode\n",
    "    try:\n",
    "        email_dict['body'] = re.sub(\"\"\"(<NONASCII>)\"\"\", ' ', email_dict['body'])\n",
    "        email_dict['body'] =  email_dict['body'].encode(\"ascii\")\n",
    "    except: \n",
    "        None\n",
    "    # lowercase all \n",
    "    email_dict['body'] = email_dict['body'].lower()\n",
    "    return email_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def ValidEmail(email_dict):\n",
    "    cond = []\n",
    "    ## Excludes email forwarding \n",
    "    cond.append(\"\"\"Forwarded by\"\"\" not in email_dict['body'])\n",
    "    ## Remove HTML emails\n",
    "    ## TODO\n",
    "    return min(cond)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "email_prepross = [\n",
    "    PreProcessor(email) for email in emails\n",
    "    if ValidEmail(email)\n",
    "    ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "    Only considers Email that have more than 20 non white space words\n",
    "\"\"\"\n",
    "\n",
    "def ValidPreprocessEmail(email_dict):\n",
    "    cond = [] \n",
    "    cond.append(len([item for item in email_dict['body'].split(\" \") if item != \" \"]) > 20)\n",
    "    return min(cond)\n",
    "\n",
    "validPreprocessed = [email for email in email_prepross if ValidPreprocessEmail(email)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "print \"Total Emails\", len(emails)\n",
    "print \"Emails with Original Message in body\", len([email for email in emails if \"Original Message\" in email['body']])\n",
    "print \"Emails with Subject in Body\", len([email for email in emails if \"Subject:\" in email['body']])\n",
    "print \"Forward Emails:\", len([email for email in emails if \"FW:\" in email['body']])\n",
    "print \"Preprocessed Emails:\", len(email_prepross)\n",
    "print \"Valid Preprocessed Emails for Training\", len([email for email in email_prepross if ValidPreprocessEmail(email)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'meeting to discuss the current information that we receive as well as the new \\ngeneration database\\n\\nthe meeting will take place in  b'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import random\n",
    "random.choice(validPreprocessed)['body']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "pickle.dump( validPreprocessed, open( \"validPreprocessed.p\", \"wb\" ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "validPreprocessed = pickle.load(open( \"validPreprocessed.p\", \"rb\" ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def WordProcessor(input_text):\n",
    "    return None\n",
    "\n",
    "\"\"\"\n",
    "    Tokenizes and Generate word Frequencies for a given text input\n",
    "\"\"\"\n",
    "def getFrequencies(input_text):\n",
    "    import nltk\n",
    "    from nltk.probability import FreqDist\n",
    "    #nltk.download()\n",
    "    tokens = nltk.word_tokenize(input_text)\n",
    "    return FreqDist(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "    Obtains word frequencies in the Enron Email Corpus\n",
    "\"\"\"\n",
    "\n",
    "from nltk.probability import FreqDist\n",
    "init_FreqDist = FreqDist()\n",
    "for email in validPreprocessed:\n",
    "    try:\n",
    "        init_FreqDist.update(getFrequencies(email['body']))\n",
    "    except:\n",
    "        print(\"error\", email['body'])\n",
    "        print \"unicode\", isinstance(email['body'], unicode)\n",
    "        print \"str\", isinstance(email['body'], str)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "pickle.dump( init_FreqDist, open( \"init_FreqDist.p\", \"wb\" ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "reload(sys)  # Reload is a hack\n",
    "sys.setdefaultencoding('UTF8')\n",
    "\n",
    "\"\"\"\n",
    "        Windsorize the dataset to exclude from consideration \n",
    "        The most frequent and infrequent 5% of words in the dataset\n",
    "\"\"\"\n",
    "num_words = len(init_FreqDist.most_common())\n",
    "start_wcnt = num_words / 20\n",
    "end_wcnt = num_words - start_wcnt\n",
    "words_dataset = init_FreqDist.most_common()[start_wcnt:end_wcnt]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#words_dataset[:100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "isinstance(validPreprocessed[200]['body'], unicode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle.dump( words_dataset, open( \"words_dataset.p\", \"wb\" ) )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(validPreprocessed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}