EnronProject.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Enron Project"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\MariaElisabeth\\Anaconda2\\lib\\site-packages\\sklearn\\cross_validation.py:44: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n",
      "C:\\Users\\MariaElisabeth\\Anaconda2\\lib\\site-packages\\sklearn\\grid_search.py:43: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. This module will be removed in 0.20.\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "#!/usr/bin/python\n",
    "from time import time\n",
    "import sys\n",
    "import pickle\n",
    "import pprint\n",
    "pp = pprint.PrettyPrinter(indent=4)\n",
    "\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.cross_validation import train_test_split\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "\n",
    "\n",
    "sys.path.append(\"../tools/\")\n",
    "\n",
    "from feature_format import featureFormat, targetFeatureSplit\n",
    "from tester import dump_classifier_and_data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "### Load the dictionary containing the dataset\n",
    "with open(\"final_project_dataset.pkl\", \"r\") as data_file:\n",
    "    data_dict = pickle.load(data_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of  people: 146\n",
      "Number of features: 21\n",
      "\n",
      "Number of POI in poi_names.txt: 35\n",
      "Number of POI in dataset: 18\n"
     ]
    }
   ],
   "source": [
    "### Explore features of data\n",
    "\n",
    "print \"Number of  people: %d\" % (len(data_dict))\n",
    "print \"Number of features: %d\" % (len(data_dict['METTS MARK']))\n",
    "print\n",
    "poi_names = open(\"../final_project/poi_names.txt\").read().split('\\n')\n",
    "poi_y = [name for name in poi_names if \"(y)\" in name]\n",
    "poi_n = [name for name in poi_names if \"(n)\" in name]\n",
    "print \"Number of POI in poi_names.txt: %d\" % len(poi_y + poi_n)\n",
    "\n",
    "poi_count = 0\n",
    "for person in data_dict:\n",
    "\tif data_dict[person][\"poi\"]==1:\n",
    "\t\tpoi_count +=1 \n",
    "print \"Number of POI in dataset: %d\" % (poi_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def neg_nan_count(data_dict):\n",
    "    '''returns a dictionary containing the number of NaNs for each feature and the number of negative numbers for each feature'''\n",
    "    keys_w_nans = dict((key, 0) for key, value in data_dict['METTS MARK'].iteritems())\n",
    "    keys_w_negs = dict((key, 0) for key, value in data_dict['METTS MARK'].iteritems())\n",
    "    for person in data_dict:\n",
    "        for key, value in data_dict[person].iteritems():\n",
    "            if value == \"NaN\":\n",
    "                keys_w_nans[key] += 1\n",
    "            elif value < 0:\n",
    "                keys_w_negs[key] += 1\n",
    "    return keys_w_nans, keys_w_negs\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of NaNs:\n",
      "{   'bonus': 64,\n",
      "    'deferral_payments': 107,\n",
      "    'deferred_income': 97,\n",
      "    'director_fees': 129,\n",
      "    'email_address': 35,\n",
      "    'exercised_stock_options': 44,\n",
      "    'expenses': 51,\n",
      "    'from_messages': 60,\n",
      "    'from_poi_to_this_person': 60,\n",
      "    'from_this_person_to_poi': 60,\n",
      "    'loan_advances': 142,\n",
      "    'long_term_incentive': 80,\n",
      "    'other': 53,\n",
      "    'poi': 0,\n",
      "    'restricted_stock': 36,\n",
      "    'restricted_stock_deferred': 128,\n",
      "    'salary': 51,\n",
      "    'shared_receipt_with_poi': 60,\n",
      "    'to_messages': 60,\n",
      "    'total_payments': 21,\n",
      "    'total_stock_value': 20}\n",
      "\n",
      "Number of Negative Values\n",
      "{   'bonus': 0,\n",
      "    'deferral_payments': 1,\n",
      "    'deferred_income': 49,\n",
      "    'director_fees': 0,\n",
      "    'email_address': 0,\n",
      "    'exercised_stock_options': 0,\n",
      "    'expenses': 0,\n",
      "    'from_messages': 0,\n",
      "    'from_poi_to_this_person': 0,\n",
      "    'from_this_person_to_poi': 0,\n",
      "    'loan_advances': 0,\n",
      "    'long_term_incentive': 0,\n",
      "    'other': 0,\n",
      "    'poi': 0,\n",
      "    'restricted_stock': 1,\n",
      "    'restricted_stock_deferred': 16,\n",
      "    'salary': 0,\n",
      "    'shared_receipt_with_poi': 0,\n",
      "    'to_messages': 0,\n",
      "    'total_payments': 0,\n",
      "    'total_stock_value': 1}\n"
     ]
    }
   ],
   "source": [
    "keys_w_nans, keys_w_negs = neg_nan_count(data_dict)\n",
    "\n",
    "print \"Number of NaNs:\"          \n",
    "pp.pprint(keys_w_nans)\n",
    "print\n",
    "print \"Number of Negative Values\"\n",
    "pp.pprint(keys_w_negs) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-102500\n",
      "BELFER ROBERT\n"
     ]
    }
   ],
   "source": [
    "for person in data_dict:\n",
    "    if data_dict[person]['deferral_payments'] < 0:\n",
    "        print data_dict[person]['deferral_payments']\n",
    "        print person"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This must be a mis-entry into the dataset. According to the pdf with financial values, this should be his 'deferred income' value, which makes sense. Deferral payments should be positive not negative.  Taking a look at all of his values (below) shows some other errors.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{   'bonus': 'NaN',\n",
      "    'deferral_payments': -102500,\n",
      "    'deferred_income': 'NaN',\n",
      "    'director_fees': 3285,\n",
      "    'email_address': 'NaN',\n",
      "    'exercised_stock_options': 3285,\n",
      "    'expenses': 'NaN',\n",
      "    'from_messages': 'NaN',\n",
      "    'from_poi_to_this_person': 'NaN',\n",
      "    'from_this_person_to_poi': 'NaN',\n",
      "    'loan_advances': 'NaN',\n",
      "    'long_term_incentive': 'NaN',\n",
      "    'other': 'NaN',\n",
      "    'poi': False,\n",
      "    'restricted_stock': 'NaN',\n",
      "    'restricted_stock_deferred': 44093,\n",
      "    'salary': 'NaN',\n",
      "    'shared_receipt_with_poi': 'NaN',\n",
      "    'to_messages': 'NaN',\n",
      "    'total_payments': 102500,\n",
      "    'total_stock_value': -44093}\n"
     ]
    }
   ],
   "source": [
    "pp.pprint (data_dict['BELFER ROBERT'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_dict['BELFER ROBERT']['director_fees'] = 102500\n",
    "data_dict['BELFER ROBERT']['deferred_income'] = -102500\n",
    "data_dict['BELFER ROBERT']['deferral_payments'] = 'NaN'\n",
    "data_dict['BELFER ROBERT']['expenses'] = 3285 \n",
    "data_dict['BELFER ROBERT']['total_payments'] = 102500\n",
    "data_dict['BELFER ROBERT']['restricted_stock'] = 44093\n",
    "data_dict['BELFER ROBERT']['restricted_stock_deferred'] = -44093\n",
    "data_dict['BELFER ROBERT']['total_stock_value'] = \"NaN\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-2604490\n",
      "BHATNAGAR SANJAY\n"
     ]
    }
   ],
   "source": [
    "for person in data_dict:\n",
    "    if data_dict[person]['restricted_stock'] < 0:\n",
    "        print data_dict[person]['restricted_stock']\n",
    "        print person"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This too is a mis-entry according to the pdf. This negative value is meant to be his 'restricted_stock_deferred' and he is mean to have a positive value here."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{   'bonus': 'NaN',\n",
      "    'deferral_payments': 'NaN',\n",
      "    'deferred_income': 'NaN',\n",
      "    'director_fees': 137864,\n",
      "    'email_address': 'sanjay.bhatnagar@enron.com',\n",
      "    'exercised_stock_options': 2604490,\n",
      "    'expenses': 'NaN',\n",
      "    'from_messages': 29,\n",
      "    'from_poi_to_this_person': 0,\n",
      "    'from_this_person_to_poi': 1,\n",
      "    'loan_advances': 'NaN',\n",
      "    'long_term_incentive': 'NaN',\n",
      "    'other': 137864,\n",
      "    'poi': False,\n",
      "    'restricted_stock': -2604490,\n",
      "    'restricted_stock_deferred': 15456290,\n",
      "    'salary': 'NaN',\n",
      "    'shared_receipt_with_poi': 463,\n",
      "    'to_messages': 523,\n",
      "    'total_payments': 15456290,\n",
      "    'total_stock_value': 'NaN'}\n"
     ]
    }
   ],
   "source": [
    "pp.pprint (data_dict['BHATNAGAR SANJAY'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There are a few corrections to be made here. This occured because there is a blank space instead of dash for 'other' meaning that all of the values got slid one to the left."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_dict['BHATNAGAR SANJAY']['exercised_stock_options'] = 15456290\n",
    "data_dict['BHATNAGAR SANJAY']['restricted_stock'] = 2604490\n",
    "data_dict['BHATNAGAR SANJAY']['restricted_stock_deferred'] = -2604490\n",
    "data_dict['BHATNAGAR SANJAY']['total_stock_value'] = 15456290\n",
    "data_dict['BHATNAGAR SANJAY']['total_payments'] = 137864\n",
    "data_dict['BHATNAGAR SANJAY']['other'] = 'NaN'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Negative Values\n",
      "{   'bonus': 0,\n",
      "    'deferral_payments': 0,\n",
      "    'deferred_income': 50,\n",
      "    'director_fees': 0,\n",
      "    'email_address': 0,\n",
      "    'exercised_stock_options': 0,\n",
      "    'expenses': 0,\n",
      "    'from_messages': 0,\n",
      "    'from_poi_to_this_person': 0,\n",
      "    'from_this_person_to_poi': 0,\n",
      "    'loan_advances': 0,\n",
      "    'long_term_incentive': 0,\n",
      "    'other': 0,\n",
      "    'poi': 0,\n",
      "    'restricted_stock': 0,\n",
      "    'restricted_stock_deferred': 18,\n",
      "    'salary': 0,\n",
      "    'shared_receipt_with_poi': 0,\n",
      "    'to_messages': 0,\n",
      "    'total_payments': 0,\n",
      "    'total_stock_value': 0}\n"
     ]
    }
   ],
   "source": [
    "keys_w_nans, keys_w_negs = neg_nan_count(data_dict)\n",
    "\n",
    "print \"Number of Negative Values\"\n",
    "pp.pprint(keys_w_negs) \n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This looks a lot cleaner now.  Let's make sure there are no negative values in either of the categories where everything should be postiive."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for person in data_dict:\n",
    "    if data_dict[person]['deferred_income'] > 0 and data_dict[person]['deferred_income'] != \"NaN\":\n",
    "        print person, data_dict[person]['deferred_income']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for person in data_dict:\n",
    "    if data_dict[person]['restricted_stock_deferred'] > 0 and data_dict[person]['restricted_stock_deferred'] != \"NaN\":\n",
    "        print person, data_dict[person]['restricted_stock_deferred']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Phew, all clear.  Let's turn those negative values into positive values now, since some classifiers (e.g. SelectKbest with chi-squared) can have issues with negative values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for person in data_dict:\n",
    "    if data_dict[person]['deferred_income'] < 0 and data_dict[person]['deferred_income'] != \"NaN\":\n",
    "        data_dict[person]['deferred_income'] = - data_dict[person]['deferred_income']\n",
    "        \n",
    "for person in data_dict:\n",
    "    if data_dict[person]['restricted_stock_deferred'] < 0 and data_dict[person]['restricted_stock_deferred'] != \"NaN\":\n",
    "        data_dict[person]['restricted_stock_deferred'] = - data_dict[person]['restricted_stock_deferred']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Negative Values\n",
      "{   'bonus': 0,\n",
      "    'deferral_payments': 0,\n",
      "    'deferred_income': 0,\n",
      "    'director_fees': 0,\n",
      "    'email_address': 0,\n",
      "    'exercised_stock_options': 0,\n",
      "    'expenses': 0,\n",
      "    'from_messages': 0,\n",
      "    'from_poi_to_this_person': 0,\n",
      "    'from_this_person_to_poi': 0,\n",
      "    'loan_advances': 0,\n",
      "    'long_term_incentive': 0,\n",
      "    'other': 0,\n",
      "    'poi': 0,\n",
      "    'restricted_stock': 0,\n",
      "    'restricted_stock_deferred': 0,\n",
      "    'salary': 0,\n",
      "    'shared_receipt_with_poi': 0,\n",
      "    'to_messages': 0,\n",
      "    'total_payments': 0,\n",
      "    'total_stock_value': 0}\n"
     ]
    }
   ],
   "source": [
    "keys_w_nans, keys_w_negs = neg_nan_count(data_dict)\n",
    "\n",
    "print \"Number of Negative Values\"\n",
    "pp.pprint(keys_w_negs) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'bonus': 'NaN',\n",
       " 'deferral_payments': 'NaN',\n",
       " 'deferred_income': 'NaN',\n",
       " 'director_fees': 'NaN',\n",
       " 'email_address': 'NaN',\n",
       " 'exercised_stock_options': 'NaN',\n",
       " 'expenses': 'NaN',\n",
       " 'from_messages': 'NaN',\n",
       " 'from_poi_to_this_person': 'NaN',\n",
       " 'from_this_person_to_poi': 'NaN',\n",
       " 'loan_advances': 'NaN',\n",
       " 'long_term_incentive': 'NaN',\n",
       " 'other': 362096,\n",
       " 'poi': False,\n",
       " 'restricted_stock': 'NaN',\n",
       " 'restricted_stock_deferred': 'NaN',\n",
       " 'salary': 'NaN',\n",
       " 'shared_receipt_with_poi': 'NaN',\n",
       " 'to_messages': 'NaN',\n",
       " 'total_payments': 362096,\n",
       " 'total_stock_value': 'NaN'}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Remove these values from the dictionary\n",
    "data_dict.pop('TOTAL')\n",
    "data_dict.pop('THE TRAVEL AGENCY IN THE PARK')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "### Feature selection attempts\n",
    "\n",
    "#Create a numerical dataframe\n",
    "import pandas as pd\n",
    "\n",
    "df = pd.DataFrame.from_dict(data_dict)\n",
    "df = df.transpose()\n",
    "df = df.drop('email_address', 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "def iqr_outliers(column):\n",
    "    '''Takes a panda series, identifies number of NaNs, removes NaNs, calculates IQR, \n",
    "    identifies low outliers (< Q1 - 1.5*IQR), high outliers (> Q3 + 1.5*IQR )\n",
    "    and returns a dictionary of the number of high, low, non-outlier values and NaNs'''\n",
    "    \n",
    "    nan_count = 0\n",
    "    neg_count = 0\n",
    "    for value in column:\n",
    "        if value == 'NaN':\n",
    "            nan_count += 1\n",
    "        elif value < 0:\n",
    "            neg_count += 1\n",
    "    cleaned_column = [x for x in column if str(x) != 'NaN']\n",
    "    iqr = np.subtract(*np.percentile(cleaned_column, [75, 25]))\n",
    "    upper =  np.percentile(cleaned_column, 75) + 1.5 * iqr\n",
    "    lower =  np.percentile(cleaned_column, 25) - 1.5 * iqr\n",
    "    outliers = []\n",
    "    lows = 0\n",
    "    highs = 0\n",
    "    non_outliers= 0\n",
    "    for value in cleaned_column:\n",
    "        if value < lower:\n",
    "            lows += 1\n",
    "            outliers.append(value)\n",
    "        elif value > upper:\n",
    "            highs += 1\n",
    "            outliers.append(value)\n",
    "        else:\n",
    "            non_outliers += 1\n",
    "                \n",
    "    \n",
    "    return ({\"Low_outliers\": lows, \n",
    "             \"High_outliers\": highs, \n",
    "             \"NaNs\": nan_count,\n",
    "             \"Negative values\": neg_count,\n",
    "            \"Non_outliers\": non_outliers\n",
    "            })\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                           High_outliers  Low_outliers  NaNs  Negative values  \\\n",
      "bonus                                 10             0    63                0   \n",
      "deferral_payments                      6             0   107                0   \n",
      "deferred_income                        5             0    95                0   \n",
      "director_fees                          1             3   128                0   \n",
      "exercised_stock_options               12             0    43                0   \n",
      "expenses                               3             0    49                0   \n",
      "from_messages                         17             0    58                0   \n",
      "from_poi_to_this_person               11             0    58                0   \n",
      "from_this_person_to_poi               13             0    58                0   \n",
      "loan_advances                          0             0   141                0   \n",
      "long_term_incentive                    7             0    79                0   \n",
      "other                                 11             0    54                0   \n",
      "poi                                   18             0     0                0   \n",
      "restricted_stock                      14             0    34                0   \n",
      "restricted_stock_deferred              2             0   127                0   \n",
      "salary                                 6             3    50                0   \n",
      "shared_receipt_with_poi                2             0    58                0   \n",
      "to_messages                            7             0    58                0   \n",
      "total_payments                         9             0    21                0   \n",
      "total_stock_value                     15             0    19                0   \n",
      "\n",
      "                           Non_outliers  \n",
      "bonus                                71  \n",
      "deferral_payments                    31  \n",
      "deferred_income                      44  \n",
      "director_fees                        12  \n",
      "exercised_stock_options              89  \n",
      "expenses                             92  \n",
      "from_messages                        69  \n",
      "from_poi_to_this_person              75  \n",
      "from_this_person_to_poi              73  \n",
      "loan_advances                         3  \n",
      "long_term_incentive                  58  \n",
      "other                                79  \n",
      "poi                                 126  \n",
      "restricted_stock                     96  \n",
      "restricted_stock_deferred            15  \n",
      "salary                               85  \n",
      "shared_receipt_with_poi              84  \n",
      "to_messages                          79  \n",
      "total_payments                      114  \n",
      "total_stock_value                   110  \n"
     ]
    }
   ],
   "source": [
    "outlier_count = dict((name, {}) for name in df.columns)\n",
    "\n",
    "for key, value in outlier_count.iteritems():\n",
    "    value = iqr_outliers(df[key])\n",
    "    outlier_count[key] = value\n",
    "\n",
    "outlier_df = pd.DataFrame.from_dict(outlier_count)\n",
    "outlier_df = outlier_df.transpose()\n",
    "\n",
    "print outlier_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature Creation and Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "### Task 1: Select what features you'll use.\n",
    "### features_list is a list of strings, each of which is a feature name.\n",
    "### The first feature must be \"poi\".\n",
    "### Currently, I have all of them selected\n",
    "\n",
    "features_list = ['poi','salary', 'to_messages', 'deferral_payments',\n",
    "                 'total_payments', 'exercised_stock_options', 'bonus', 'restricted_stock',\n",
    "                 'shared_receipt_with_poi', 'restricted_stock_deferred', 'total_stock_value',\n",
    "                 'expenses', 'loan_advances', 'from_messages', 'other', 'from_this_person_to_poi',\n",
    "                 'director_fees', 'deferred_income', 'long_term_incentive', 'from_poi_to_this_person']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create the following features:\n",
    ">- Percent salary/total payments\n",
    ">- Percent bonus/total payments\n",
    ">- Ratio of salary:bonus\n",
    ">- Ratio of total stock value:total payments\n",
    ">- Percent excercised stock/total stock value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def create_ratio(data_dict, ratio_name, numerator, denominator):\n",
    "    '''Calcultes the ratio between a given numerator and denominator\n",
    "    Names the ratio \"ratio_name\"\n",
    "    Returns the the updated dictionary with the ratio values'''\n",
    "    \n",
    "    for person in data_dict:\n",
    "        if data_dict[person][numerator] == 'NaN' or data_dict[person][denominator] == 'NaN':\n",
    "                data_dict[person][ratio_name] = 'NaN'\n",
    "        else:\n",
    "            data_dict[person][ratio_name] = float(data_dict[person][numerator])/float(data_dict[person][denominator])\n",
    "    return data_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data_dict = create_ratio(data_dict, 'sal_total', 'salary', 'total_payments')\n",
    "data_dict = create_ratio(data_dict, 'bon_total', 'bonus', 'total_payments')\n",
    "data_dict = create_ratio(data_dict, 'sal_bon', 'salary', 'bonus')\n",
    "data_dict = create_ratio(data_dict, 'stock_pay', 'total_stock_value', 'total_payments')\n",
    "data_dict = create_ratio(data_dict, 'excer_stock', 'exercised_stock_options', 'total_stock_value')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "features_list.append('sal_total')\n",
    "features_list.append('bon_total')\n",
    "features_list.append('sal_bon')\n",
    "features_list.append('stock_pay')\n",
    "features_list.append('excer_stock')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "enron_df = pd.DataFrame.from_dict(data_dict)\n",
    "enron_df = enron_df.transpose()\n",
    "enron_df = enron_df.drop('email_address', 1)\n",
    "\n",
    "#Right now NaNs are string, not actual NaNs -- change this\n",
    "enron_df.replace(['NaN'], [None], inplace=True)\n",
    "\n",
    "#Creates an alternate dataframe with NaNs replaced by 0s\n",
    "no_nans = enron_df.fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_selection import VarianceThreshold\n",
    "\n",
    "#Use Variance Threshold to select features with the highest variance\n",
    "\n",
    "sel = VarianceThreshold(threshold=(.8 * (1 - .8)))\n",
    "sel.fit_transform(no_nans)\n",
    "indices = sel.get_support()\n",
    "\n",
    "discarded = []\n",
    "for i in range(0, len(indices)):\n",
    "    if indices[i] == False:\n",
    "        discarded.append(list(no_nans.columns.values)[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['excer_stock', 'poi', 'sal_total']"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "discarded"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Oh, this is rich.  After all that work (it was short code, but my basic programming skills made this take a while), the only feature with low variance is poi, which we would remove anyway and two of the combined features.  Let's see what other columns might have relatively low variance when we change the threshold."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['bon_total', 'excer_stock', 'from_messages', 'from_poi_to_this_person', 'from_this_person_to_poi', 'poi', 'sal_bon', 'sal_total', 'shared_receipt_with_poi', 'stock_pay', 'to_messages']\n"
     ]
    }
   ],
   "source": [
    "def usevariance_t(thresh, df):\n",
    "    '''Takes a numerical threshold and a dataframe\n",
    "    Implements the VarianceThreshold method for feature selection\n",
    "    Returns a list of features which were removed because their variance was lower than the threshold'''\n",
    "    sel = VarianceThreshold(threshold=thresh)\n",
    "    reduced = sel.fit(df)\n",
    "    indices = sel.get_support()\n",
    "    features = []\n",
    "    for i in range(0, len(indices)):\n",
    "        if indices[i] == False:\n",
    "            features.append(list(no_nans.columns.values)[i])\n",
    "    return features\n",
    "\n",
    "print usevariance_t(10000000, no_nans)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It makes sense that the other columns with relatively low variance are those dealing with e-mails and the other ratios.  That makes sense because those numbers IN GENERAL will be lower than salaries and thus variance will be lower. More and more it seems to me that the whole variance threshold thing will only really work with normalized features in each column."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['loan_advances', 'stock_pay', 'total_payments']\n"
     ]
    }
   ],
   "source": [
    "from sklearn import preprocessing\n",
    "\n",
    "min_max_scaler = preprocessing.MinMaxScaler()\n",
    "df_scaled = min_max_scaler.fit_transform(no_nans)\n",
    "df_normalized = pd.DataFrame(df_scaled)\n",
    "\n",
    "print usevariance_t(.01, df_normalized)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Ok, so this might be more meaningful.  It looks like there is relatively low variance (< 0.01) within these categories.  Not sure if that means I should eliminate them yet. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Splits the dataframe into a series (the poi column) and the rest of the dataframe\n",
    "poi_labels = enron_df['poi']\n",
    "no_pois = enron_df.drop('poi', 1)\n",
    "no_pois = no_pois.replace(\"NaN\", 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ALLEN PHILLIP K       False\n",
       "BADUM JAMES P         False\n",
       "BANNANTINE JAMES M    False\n",
       "BAXTER JOHN C         False\n",
       "BAY FRANKLIN R        False\n",
       "Name: poi, dtype: bool"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "poi_labels.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bon_total</th>\n",
       "      <th>bonus</th>\n",
       "      <th>deferral_payments</th>\n",
       "      <th>deferred_income</th>\n",
       "      <th>director_fees</th>\n",
       "      <th>excer_stock</th>\n",
       "      <th>exercised_stock_options</th>\n",
       "      <th>expenses</th>\n",
       "      <th>from_messages</th>\n",
       "      <th>from_poi_to_this_person</th>\n",
       "      <th>...</th>\n",
       "      <th>restricted_stock</th>\n",
       "      <th>restricted_stock_deferred</th>\n",
       "      <th>sal_bon</th>\n",
       "      <th>sal_total</th>\n",
       "      <th>salary</th>\n",
       "      <th>shared_receipt_with_poi</th>\n",
       "      <th>stock_pay</th>\n",
       "      <th>to_messages</th>\n",
       "      <th>total_payments</th>\n",
       "      <th>total_stock_value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ALLEN PHILLIP K</th>\n",
       "      <td>0.930997</td>\n",
       "      <td>4175000.0</td>\n",
       "      <td>2869717.0</td>\n",
       "      <td>3081055.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1729541.0</td>\n",
       "      <td>13868.0</td>\n",
       "      <td>2195.0</td>\n",
       "      <td>47.0</td>\n",
       "      <td>...</td>\n",
       "      <td>126027.0</td>\n",
       "      <td>126027.0</td>\n",
       "      <td>0.048372</td>\n",
       "      <td>0.045035</td>\n",
       "      <td>201955.0</td>\n",
       "      <td>1407.0</td>\n",
       "      <td>0.385676</td>\n",
       "      <td>2902.0</td>\n",
       "      <td>4484442.0</td>\n",
       "      <td>1729541.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BADUM JAMES P</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>178980.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>257817.0</td>\n",
       "      <td>3486.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.412959</td>\n",
       "      <td>0.0</td>\n",
       "      <td>182466.0</td>\n",
       "      <td>257817.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BANNANTINE JAMES M</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5104.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.771654</td>\n",
       "      <td>4046157.0</td>\n",
       "      <td>56301.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>39.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1757552.0</td>\n",
       "      <td>560222.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000521</td>\n",
       "      <td>477.0</td>\n",
       "      <td>465.0</td>\n",
       "      <td>5.723100</td>\n",
       "      <td>566.0</td>\n",
       "      <td>916197.0</td>\n",
       "      <td>5243487.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BAXTER JOHN C</th>\n",
       "      <td>0.212980</td>\n",
       "      <td>1200000.0</td>\n",
       "      <td>1295738.0</td>\n",
       "      <td>1386055.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.628860</td>\n",
       "      <td>6680544.0</td>\n",
       "      <td>11200.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>3942714.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.222585</td>\n",
       "      <td>0.047406</td>\n",
       "      <td>267102.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.885448</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5634343.0</td>\n",
       "      <td>10623258.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BAY FRANKLIN R</th>\n",
       "      <td>0.483269</td>\n",
       "      <td>400000.0</td>\n",
       "      <td>260455.0</td>\n",
       "      <td>201641.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>129142.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>145796.0</td>\n",
       "      <td>82782.0</td>\n",
       "      <td>0.599178</td>\n",
       "      <td>0.289564</td>\n",
       "      <td>239671.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.076132</td>\n",
       "      <td>0.0</td>\n",
       "      <td>827696.0</td>\n",
       "      <td>63014.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                    bon_total      bonus  deferral_payments  deferred_income  \\\n",
       "ALLEN PHILLIP K      0.930997  4175000.0          2869717.0        3081055.0   \n",
       "BADUM JAMES P        0.000000        0.0           178980.0              0.0   \n",
       "BANNANTINE JAMES M   0.000000        0.0                0.0           5104.0   \n",
       "BAXTER JOHN C        0.212980  1200000.0          1295738.0        1386055.0   \n",
       "BAY FRANKLIN R       0.483269   400000.0           260455.0         201641.0   \n",
       "\n",
       "                    director_fees  excer_stock  exercised_stock_options  \\\n",
       "ALLEN PHILLIP K               0.0     1.000000                1729541.0   \n",
       "BADUM JAMES P                 0.0     1.000000                 257817.0   \n",
       "BANNANTINE JAMES M            0.0     0.771654                4046157.0   \n",
       "BAXTER JOHN C                 0.0     0.628860                6680544.0   \n",
       "BAY FRANKLIN R                0.0     0.000000                      0.0   \n",
       "\n",
       "                    expenses  from_messages  from_poi_to_this_person  \\\n",
       "ALLEN PHILLIP K      13868.0         2195.0                     47.0   \n",
       "BADUM JAMES P         3486.0            0.0                      0.0   \n",
       "BANNANTINE JAMES M   56301.0           29.0                     39.0   \n",
       "BAXTER JOHN C        11200.0            0.0                      0.0   \n",
       "BAY FRANKLIN R      129142.0            0.0                      0.0   \n",
       "\n",
       "                          ...          restricted_stock  \\\n",
       "ALLEN PHILLIP K           ...                  126027.0   \n",
       "BADUM JAMES P             ...                       0.0   \n",
       "BANNANTINE JAMES M        ...                 1757552.0   \n",
       "BAXTER JOHN C             ...                 3942714.0   \n",
       "BAY FRANKLIN R            ...                  145796.0   \n",
       "\n",
       "                    restricted_stock_deferred   sal_bon  sal_total    salary  \\\n",
       "ALLEN PHILLIP K                      126027.0  0.048372   0.045035  201955.0   \n",
       "BADUM JAMES P                             0.0  0.000000   0.000000       0.0   \n",
       "BANNANTINE JAMES M                   560222.0  0.000000   0.000521     477.0   \n",
       "BAXTER JOHN C                             0.0  0.222585   0.047406  267102.0   \n",
       "BAY FRANKLIN R                        82782.0  0.599178   0.289564  239671.0   \n",
       "\n",
       "                    shared_receipt_with_poi  stock_pay  to_messages  \\\n",
       "ALLEN PHILLIP K                      1407.0   0.385676       2902.0   \n",
       "BADUM JAMES P                           0.0   1.412959          0.0   \n",
       "BANNANTINE JAMES M                    465.0   5.723100        566.0   \n",
       "BAXTER JOHN C                           0.0   1.885448          0.0   \n",
       "BAY FRANKLIN R                          0.0   0.076132          0.0   \n",
       "\n",
       "                    total_payments  total_stock_value  \n",
       "ALLEN PHILLIP K          4484442.0          1729541.0  \n",
       "BADUM JAMES P             182466.0           257817.0  \n",
       "BANNANTINE JAMES M        916197.0          5243487.0  \n",
       "BAXTER JOHN C            5634343.0         10623258.0  \n",
       "BAY FRANKLIN R            827696.0            63014.0  \n",
       "\n",
       "[5 rows x 24 columns]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Check to make sure no poi or e-mail columns and no NaNs\n",
    "no_pois.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "\n",
    "clf = DecisionTreeClassifier()\n",
    "clf = clf.fit(no_pois, poi_labels)\n",
    "imp_dic = {}\n",
    "for feature, importance in zip(list(no_pois.columns.values), clf.feature_importances_):\n",
    "    imp_dic[feature] = importance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'bon_total': 0.0,\n",
       " 'bonus': 0.10945152803076318,\n",
       " 'deferral_payments': 0.0,\n",
       " 'deferred_income': 0.0,\n",
       " 'director_fees': 0.042328042328042326,\n",
       " 'excer_stock': 0.0,\n",
       " 'exercised_stock_options': 0.19999999999999979,\n",
       " 'expenses': 0.064965420713307881,\n",
       " 'from_messages': 0.057720057720057727,\n",
       " 'from_poi_to_this_person': 0.097883597883597878,\n",
       " 'from_this_person_to_poi': 0.0073375262054505997,\n",
       " 'loan_advances': 0.0,\n",
       " 'long_term_incentive': 0.10884353741496598,\n",
       " 'other': 0.029993455569975496,\n",
       " 'restricted_stock': 0.10991999591033426,\n",
       " 'restricted_stock_deferred': 0.0,\n",
       " 'sal_bon': 0.0,\n",
       " 'sal_total': 0.12194507432602666,\n",
       " 'salary': 0.0,\n",
       " 'shared_receipt_with_poi': 0.0,\n",
       " 'stock_pay': 0.04961176389747822,\n",
       " 'to_messages': 0.0,\n",
       " 'total_payments': 0.0,\n",
       " 'total_stock_value': 0.0}"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imp_dic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_selection import SelectKBest\n",
    "from sklearn.feature_selection import chi2\n",
    "from sklearn.feature_selection import mutual_info_classif\n",
    "\n",
    "def use_KBest(non_neg_df, score, feat_num):\n",
    "    '''Uses the KBest feature selection\n",
    "    Takes as input: a dataframe with non-negative values, which score you would like to use and number of features\n",
    "    Returns a dictionary k_tops with the most to least important features, (key = place, value = feature name)\n",
    "    Returns a dictionary k_scores of actualy scores '''\n",
    "    #Iterates through selecting 1st best, 2nd best, etc. feature to create a dictionary \n",
    "    #where key is a number and value is the feature\n",
    "\n",
    "    k_tops = {}\n",
    "    for i in range(1, feat_num+1):\n",
    "        ch2 = SelectKBest(score, k = i)\n",
    "        ch2.fit(non_neg_df, poi_labels)\n",
    "        klist = ch2.get_support()\n",
    "        for truth, name in zip(klist, list(non_neg_df.columns.values)):\n",
    "            if truth and name not in k_tops.values():\n",
    "                k_tops[i] = (name)\n",
    "\n",
    "    ch2 = SelectKBest(score, k = 'all')\n",
    "    ch2.fit(non_neg_df, poi_labels)\n",
    "    scores = ch2.scores_\n",
    "\n",
    "    k_scores = {}\n",
    "    for score, name in zip(scores, list(non_neg_df.columns.values)):\n",
    "        k_scores[name] = score\n",
    "    \n",
    "    return k_tops, k_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Calculates chi-squared scores in KBest\n",
    "chi_k_tops, chi_k_scores = use_KBest(no_pois, chi2, len(no_pois.columns))\n",
    "\n",
    "#Calculates mutual-information score \n",
    "mut_k_tops, mut_k_scores = use_KBest(no_pois, mutual_info_classif, len(no_pois.columns))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import operator\n",
    "ord_imp_dic = sorted(imp_dic.items(), key=operator.itemgetter(1), reverse = True)    \n",
    "ord_chi_scores = sorted(chi_k_scores.items(), key=operator.itemgetter(1), reverse = True)\n",
    "ord_mut_scores = sorted(mut_k_scores.items(), key=operator.itemgetter(1), reverse = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('loan_advances', 549702499.04251242),\n",
       " ('total_payments', 317800121.982463),\n",
       " ('total_stock_value', 257709136.55456081),\n",
       " ('exercised_stock_options', 219904009.58211353),\n",
       " ('bonus', 41546794.079927064),\n",
       " ('restricted_stock', 37520984.091158733),\n",
       " ('deferred_income', 20328959.706663936),\n",
       " ('other', 18156107.284525376),\n",
       " ('long_term_incentive', 13273623.898099067),\n",
       " ('salary', 3463395.416699328),\n",
       " ('restricted_stock_deferred', 1082398.2857142857),\n",
       " ('deferral_payments', 567318.57663510973),\n",
       " ('expenses', 348139.58355496643),\n",
       " ('director_fees', 219483.0),\n",
       " ('shared_receipt_with_poi', 13704.817630381005),\n",
       " ('to_messages', 6833.8754052980303),\n",
       " ('from_messages', 955.78800082948601),\n",
       " ('from_poi_to_this_person', 738.41454424450308),\n",
       " ('from_this_person_to_poi', 620.96027717347522),\n",
       " ('stock_pay', 98.333051732246957),\n",
       " ('bon_total', 18.308612008608346),\n",
       " ('sal_total', 0.74596712680387356),\n",
       " ('excer_stock', 0.019507424303642693),\n",
       " ('sal_bon', 1.3332479328604013e-06)]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ord_chi_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('bonus', 0.080438187398327976),\n",
       " ('expenses', 0.072312562666069224),\n",
       " ('other', 0.068519472897583533),\n",
       " ('excer_stock', 0.057737907628071294),\n",
       " ('shared_receipt_with_poi', 0.050474381241929045),\n",
       " ('restricted_stock_deferred', 0.044686386888540985),\n",
       " ('deferred_income', 0.042554784384563948),\n",
       " ('total_stock_value', 0.03792731667962812),\n",
       " ('sal_bon', 0.032803777834637726),\n",
       " ('from_this_person_to_poi', 0.032172151249011627),\n",
       " ('salary', 0.027706413998022539),\n",
       " ('restricted_stock', 0.027402313295444625),\n",
       " ('loan_advances', 0.025678543623489869),\n",
       " ('director_fees', 0.024465338593110175),\n",
       " ('from_poi_to_this_person', 0.01901148207318526),\n",
       " ('stock_pay', 0.015994350541096658),\n",
       " ('total_payments', 0.014427953983081387),\n",
       " ('exercised_stock_options', 0.012091285776731242),\n",
       " ('long_term_incentive', 0.0074101030719704308),\n",
       " ('from_messages', 0.00034702616094617333),\n",
       " ('to_messages', 0.0),\n",
       " ('deferral_payments', 0.0),\n",
       " ('sal_total', 0.0),\n",
       " ('bon_total', 0.0)]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ord_mut_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "### Store to my_dataset for easy export below. (This is after outlier removal (\"TOTAL\") and (\"TRAVEL AGENCY IN THE PARK\"))\n",
    "my_dataset = data_dict\n",
    "\n",
    "### Extract features and labels from dataset for local testing\n",
    "data = featureFormat(my_dataset, features_list, sort_keys = True)\n",
    "labels, features = targetFeatureSplit(data)\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "features_train, features_test, labels_train, labels_test = \\\n",
    "    train_test_split(features, labels, test_size=0.3, random_state=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Classifier Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "### Task 4: Try a varity of classifiers\n",
    "### Please name your classifier clf for easy export below.\n",
    "### Note that if you want to do PCA or other multi-stage operations,\n",
    "### you'll need to use Pipelines. For more info:\n",
    "### http://scikit-learn.org/stable/modules/pipeline.html\n",
    "\n",
    "# Out of the box algorithms\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.svm import SVC\n",
    "from sklearn import tree\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.linear_model import SGDClassifier\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def run_clf(clf, features_train, features_test, labels_train, labels_test):\n",
    "    ''' takes a classifier and training and test data\n",
    "    prints performance time and metrics'''\n",
    "    t0 = time()\n",
    "    clf.fit(features_train, labels_train)\n",
    "    print \"training time:\", round(time()-t0, 3), \"s\"\n",
    "    t0 = time()\n",
    "    labels_prediction = clf.predict(features_test)\n",
    "    print \"prediction time:\", round(time()-t0, 3), \"s\"\n",
    "    report = classification_report(labels_test, labels_prediction)\n",
    "    print report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Naive Bayes Classifier:\n",
      "training time: 0.003 s\n",
      "prediction time: 0.0 s\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "        0.0       0.92      0.95      0.94        38\n",
      "        1.0       0.50      0.40      0.44         5\n",
      "\n",
      "avg / total       0.87      0.88      0.88        43\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print \"Naive Bayes Classifier:\"\n",
    "nb_clf = GaussianNB()\n",
    "run_clf(nb_clf, features_train, features_test, labels_train, labels_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Support Vector Classifier\n",
      "training time: 0.004 s\n",
      "prediction time: 0.001 s\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "        0.0       0.88      1.00      0.94        38\n",
      "        1.0       0.00      0.00      0.00         5\n",
      "\n",
      "avg / total       0.78      0.88      0.83        43\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\MariaElisabeth\\Anaconda2\\lib\\site-packages\\sklearn\\metrics\\classification.py:1113: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "svm_clf = SVC(kernel=\"rbf\", C = 10000)\n",
    "print \"Support Vector Classifier\"\n",
    "run_clf(svm_clf, features_train, features_test, labels_train, labels_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Decision Tree\n",
      "training time: 0.003 s\n",
      "prediction time: 0.001 s\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "        0.0       0.90      0.97      0.94        38\n",
      "        1.0       0.50      0.20      0.29         5\n",
      "\n",
      "avg / total       0.86      0.88      0.86        43\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print \"Decision Tree\"\n",
    "split = tree.DecisionTreeClassifier(min_samples_split = 10)\n",
    "run_clf(split, features_train, features_test, labels_train, labels_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "K Nearest Neighbors\n",
      "training time: 0.002 s\n",
      "prediction time: 0.005 s\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "        0.0       0.93      0.97      0.95        38\n",
      "        1.0       0.67      0.40      0.50         5\n",
      "\n",
      "avg / total       0.89      0.91      0.90        43\n",
      "\n"
     ]
    }
   ],
   "source": [
    "neigh_clf = KNeighborsClassifier(n_neighbors = 3)\n",
    "print \"K Nearest Neighbors\"\n",
    "run_clf(neigh_clf, features_train, features_test, labels_train, labels_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "K Nearest Neighbors w Scaling\n",
      "training time: 0.001 s\n",
      "prediction time: 0.005 s\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "        0.0       0.92      0.95      0.94        38\n",
      "        1.0       0.50      0.40      0.44         5\n",
      "\n",
      "avg / total       0.87      0.88      0.88        43\n",
      "\n"
     ]
    }
   ],
   "source": [
    "neigh_clf = KNeighborsClassifier(n_neighbors = 3)\n",
    "print \"K Nearest Neighbors w Scaling\"\n",
    "run_clf(neigh_clf, preprocessing.MinMaxScaler().fit_transform(features_train), preprocessing.MinMaxScaler().fit_transform(features_test), labels_train, labels_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stochastic Gradient Descent\n",
      "training time: 0.004 s\n",
      "prediction time: 0.001 s\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "        0.0       0.87      0.89      0.88        38\n",
      "        1.0       0.00      0.00      0.00         5\n",
      "\n",
      "avg / total       0.77      0.79      0.78        43\n",
      "\n"
     ]
    }
   ],
   "source": [
    "sgd_clf = SGDClassifier(loss=\"log\")\n",
    "print \"Stochastic Gradient Descent\"\n",
    "run_clf(sgd_clf,(features_train), (features_test), labels_train, labels_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stochastic Gradient Descent w scaling\n",
      "training time: 0.002 s\n",
      "prediction time: 0.0 s\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "        0.0       0.91      0.82      0.86        38\n",
      "        1.0       0.22      0.40      0.29         5\n",
      "\n",
      "avg / total       0.83      0.77      0.79        43\n",
      "\n"
     ]
    }
   ],
   "source": [
    "sgd_clf = SGDClassifier(loss=\"log\")\n",
    "print \"Stochastic Gradient Descent w scaling\"\n",
    "run_clf(sgd_clf, preprocessing.MinMaxScaler().fit_transform(features_train), preprocessing.MinMaxScaler().fit_transform(features_test), labels_train, labels_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training time: 0.206 s\n",
      "prediction time: 0.044 s\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "        0.0       0.90      1.00      0.95        38\n",
      "        1.0       1.00      0.20      0.33         5\n",
      "\n",
      "avg / total       0.92      0.91      0.88        43\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "rando = RandomForestClassifier(n_estimators=10)\n",
    "run_clf(rando, features_train, features_test, labels_train, labels_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training time: 0.976 s\n",
      "prediction time: 0.0 s\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "        0.0       0.93      0.97      0.95        38\n",
      "        1.0       0.67      0.40      0.50         5\n",
      "\n",
      "avg / total       0.89      0.91      0.90        43\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "ada_clf = AdaBoostClassifier(n_estimators=100)\n",
    "run_clf(ada_clf, features_train, features_test, labels_train, labels_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        0.0       0.88      0.95      0.91        38\n",
      "        1.0       0.00      0.00      0.00         5\n",
      "\n",
      "avg / total       0.78      0.84      0.81        43\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import sklearn.pipeline\n",
    "from sklearn.decomposition import PCA\n",
    "\n",
    "scaler = preprocessing.MinMaxScaler(feature_range=(0, 1))\n",
    "select = SelectKBest(score_func = chi2, k = 10)\n",
    "pca = PCA(n_components = 5)\n",
    "kneighs = KNeighborsClassifier(n_neighbors = 3, n_jobs = -1)\n",
    "\n",
    "steps = [('scaling', scaler),\n",
    "        ('feature_selection', select),\n",
    "        ('reduce_dim', pca),\n",
    "        ('k_neighbors', kneighs)]\n",
    "\n",
    "kNN_pipeline = sklearn.pipeline.Pipeline(steps)\n",
    "kNN_pipeline.fit(features_train, labels_train)\n",
    "labels_prediction = kNN_pipeline.predict(features_test)\n",
    "report = classification_report(labels_test, labels_prediction)\n",
    "print(report)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        0.0       0.89      0.87      0.88        38\n",
      "        1.0       0.17      0.20      0.18         5\n",
      "\n",
      "avg / total       0.81      0.79      0.80        43\n",
      "\n"
     ]
    }
   ],
   "source": [
    "ada = AdaBoostClassifier(n_estimators=100)\n",
    "\n",
    "steps = [('feature_selection', select),\n",
    "         ('reduce_dim', pca),\n",
    "        ('adaboost', ada)]\n",
    "\n",
    "ada_pipeline = sklearn.pipeline.Pipeline(steps)\n",
    "ada_pipeline.fit(features_train, labels_train)\n",
    "labels_prediction = ada_pipeline.predict(features_test)\n",
    "report = classification_report(labels_test, labels_prediction)\n",
    "print(report)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Accuracy is not a good evaluation metric here because of the small number of POIs.  For example, in the testing set, there are 15 values, only one of which is a POI. Accuracy is defined at the number of items labeled correctly/total number of items. So if we were simply to make a rule to always predict non-POI, in this small testing set and with this skewed data, our accuracy would be 14/15 or .93. Edit: I updated this to show precision, recall, and f1-score instead. Also by changing the testing data to be 30% rather than 10% of the data, this allowed for more variation in the evaluation metrics.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Parameter Tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.cross_validation import StratifiedShuffleSplit\n",
    "\n",
    "parameters = dict(feature_selection__k = [5, 10, 15, 20],\n",
    "                  feature_selection__score_func = [chi2, mutual_info_classif, f_classif],\n",
    "                  reduce_dim__n_components = [1, 2, 3, 4],\n",
    "                  k_neighbors__n_neighbors = [3, 5, 7, 9],\n",
    "                  k_neighbors__n_jobs = [-1],\n",
    "                  k_neighbors__algorithm = ['ball_tree', 'kd_tree'],\n",
    "                  k_neighbors__weights = ['uniform', 'distance'],\n",
    "                  k_neighbors__leaf_size =[30, 10, 60, 100]\n",
    "               )\n",
    "\n",
    "\n",
    "cv = StratifiedShuffleSplit(labels, 100, random_state = 42)\n",
    "\n",
    "kNN_gs = GridSearchCV(kNN_pipeline, param_grid = parameters, cv = cv, scoring = 'f1')\n",
    "t0 = time()\n",
    "kNN_gs.fit(features, labels)\n",
    "print \"training time:\", round(time()-t0, 3), \"s\"\n",
    "t0 = time()\n",
    "labels_predictions = kNN_gs.predict(features)\n",
    "print \"prediction time:\", round(time()-t0, 3), \"s\"\n",
    "kNN_clf = kNN_gs.best_estimator_\n",
    "report = classification_report(labels, labels_predictions)\n",
    "print(report)\n",
    "print kNN_clf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "parameters = dict(feature_selection__k = [5, 10, 15, 20],\n",
    "                  feature_selection__score_func = [chi2, mutual_info_classif, f_classif],\n",
    "                  reduce_dim__n_components = [1, 2, 3, 4],\n",
    "                  adaboost__n_estimators = [50, 75, 100, 200], )\n",
    "\n",
    "ada_gs = GridSearchCV(ada_pipeline, param_grid = parameters, cv = cv, scoring = 'f1')\n",
    "t0 = time()\n",
    "ada_gs.fit(features, labels)\n",
    "print \"training time:\", round(time()-t0, 3), \"s\"\n",
    "t0 = time()\n",
    "labels_predictions = ada_gs.predict(features)\n",
    "print \"prediction time:\", round(time()-t0, 3), \"s\"\n",
    "ada_clf = gs.best_estimator_\n",
    "report = classification_report(labels, labels_predictions)\n",
    "print(report)\n",
    "print ada_clf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Find selected features and their scores\n",
    "\n",
    "SKB_k = SelectKBest(score_func = mutual_info_classif, k = 20)\n",
    "SKB_k.fit_transform(features, labels)   \n",
    "features_selected = [features_list[1:][i]for i in SKB_k.get_support(indices=True)]\n",
    "features_scores_selected=[feature_scores[i]for i in SKB_k.get_support(indices=True)]\n",
    "print ' '\n",
    "print 'Selected Features', features_selected\n",
    "print 'Feature Scores', features_scores_selected"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "### Task 6: Dump your classifier, dataset, and features_list so anyone can\n",
    "### check your results. You do not need to change anything below, but make sure\n",
    "### that the version of poi_id.py that you submit can be run on its own and\n",
    "### generates the necessary .pkl files for validating your results.\n",
    "\n",
    "clf = kNN_clf\n",
    "dump_classifier_and_data(clf, my_dataset, features_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from tester import test_classifier\n",
    "test_classifier(clf, my_dataset, features_list, folds = 1000)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}