Classifier.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Introduction #\n",
    "This notebook is being used for the development of a data classifier for CalPolyDnD's capstone project. The classifier needs to be able to determine what kind of data is in each column of a dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import HashingVectorizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "from sklearn.svm import LinearSVC\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import scipy\n",
    "\n",
    "import joblib as jl"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data Transformation and Cleaning ###"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The first step in solving this problem is reshaping the dataset so that we can make conclusions about each column rather than each row like in typical machine learning problems."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>first_name</th>\n",
       "      <th>last_name</th>\n",
       "      <th>email</th>\n",
       "      <th>gender</th>\n",
       "      <th>ip_address</th>\n",
       "      <th>phone_number</th>\n",
       "      <th>home_address</th>\n",
       "      <th>ssn</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Jill</td>\n",
       "      <td>Mish</td>\n",
       "      <td>jmish0@sbwire.com</td>\n",
       "      <td>Female</td>\n",
       "      <td>107.233.251.179</td>\n",
       "      <td>364-613-4322</td>\n",
       "      <td>593 Dryden Park</td>\n",
       "      <td>875-50-3238</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Frasquito</td>\n",
       "      <td>Hamer</td>\n",
       "      <td>fhamer1@washingtonpost.com</td>\n",
       "      <td>Male</td>\n",
       "      <td>150.174.167.55</td>\n",
       "      <td>613-810-5889</td>\n",
       "      <td>842 Westport Trail</td>\n",
       "      <td>109-37-5461</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Maddie</td>\n",
       "      <td>Muggleston</td>\n",
       "      <td>mmuggleston2@amazon.co.jp</td>\n",
       "      <td>Female</td>\n",
       "      <td>140.184.222.179</td>\n",
       "      <td>158-740-4562</td>\n",
       "      <td>8 Ridgeview Terrace</td>\n",
       "      <td>618-05-4668</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Selia</td>\n",
       "      <td>Wiffield</td>\n",
       "      <td>swiffield3@edublogs.org</td>\n",
       "      <td>Female</td>\n",
       "      <td>75.169.255.200</td>\n",
       "      <td>472-249-0728</td>\n",
       "      <td>4890 Acker Lane</td>\n",
       "      <td>677-86-3312</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Raleigh</td>\n",
       "      <td>Ianetti</td>\n",
       "      <td>rianetti4@weebly.com</td>\n",
       "      <td>Male</td>\n",
       "      <td>222.183.109.137</td>\n",
       "      <td>657-609-7297</td>\n",
       "      <td>12827 Barnett Plaza</td>\n",
       "      <td>235-54-2795</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  first_name   last_name                       email  gender       ip_address  \\\n",
       "0       Jill        Mish           jmish0@sbwire.com  Female  107.233.251.179   \n",
       "1  Frasquito       Hamer  fhamer1@washingtonpost.com    Male   150.174.167.55   \n",
       "2     Maddie  Muggleston   mmuggleston2@amazon.co.jp  Female  140.184.222.179   \n",
       "3      Selia    Wiffield     swiffield3@edublogs.org  Female   75.169.255.200   \n",
       "4    Raleigh     Ianetti        rianetti4@weebly.com    Male  222.183.109.137   \n",
       "\n",
       "   phone_number         home_address          ssn  \n",
       "0  364-613-4322      593 Dryden Park  875-50-3238  \n",
       "1  613-810-5889   842 Westport Trail  109-37-5461  \n",
       "2  158-740-4562  8 Ridgeview Terrace  618-05-4668  \n",
       "3  472-249-0728      4890 Acker Lane  677-86-3312  \n",
       "4  657-609-7297  12827 Barnett Plaza  235-54-2795  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"data/training_data.csv\")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To do this we transpose the data and add a new column to this new dataframe that contains the category of data that the row contained."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>1991</th>\n",
       "      <th>1992</th>\n",
       "      <th>1993</th>\n",
       "      <th>1994</th>\n",
       "      <th>1995</th>\n",
       "      <th>1996</th>\n",
       "      <th>1997</th>\n",
       "      <th>1998</th>\n",
       "      <th>1999</th>\n",
       "      <th>category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>first_name</th>\n",
       "      <td>Jill</td>\n",
       "      <td>Frasquito</td>\n",
       "      <td>Maddie</td>\n",
       "      <td>Selia</td>\n",
       "      <td>Raleigh</td>\n",
       "      <td>Kipp</td>\n",
       "      <td>Flossi</td>\n",
       "      <td>Kaia</td>\n",
       "      <td>Carolann</td>\n",
       "      <td>Cam</td>\n",
       "      <td>...</td>\n",
       "      <td>Gaston</td>\n",
       "      <td>Gennie</td>\n",
       "      <td>Keen</td>\n",
       "      <td>Myranda</td>\n",
       "      <td>Regina</td>\n",
       "      <td>Wolf</td>\n",
       "      <td>Joanna</td>\n",
       "      <td>Bianca</td>\n",
       "      <td>Jabez</td>\n",
       "      <td>first_name</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>last_name</th>\n",
       "      <td>Mish</td>\n",
       "      <td>Hamer</td>\n",
       "      <td>Muggleston</td>\n",
       "      <td>Wiffield</td>\n",
       "      <td>Ianetti</td>\n",
       "      <td>Abercromby</td>\n",
       "      <td>Volkers</td>\n",
       "      <td>Flisher</td>\n",
       "      <td>Holttom</td>\n",
       "      <td>Adacot</td>\n",
       "      <td>...</td>\n",
       "      <td>Fawdry</td>\n",
       "      <td>Guichard</td>\n",
       "      <td>McTrustram</td>\n",
       "      <td>Marchent</td>\n",
       "      <td>Aberkirdo</td>\n",
       "      <td>Bithell</td>\n",
       "      <td>Laurie</td>\n",
       "      <td>Shadrack</td>\n",
       "      <td>Pretious</td>\n",
       "      <td>last_name</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>email</th>\n",
       "      <td>jmish0@sbwire.com</td>\n",
       "      <td>fhamer1@washingtonpost.com</td>\n",
       "      <td>mmuggleston2@amazon.co.jp</td>\n",
       "      <td>swiffield3@edublogs.org</td>\n",
       "      <td>rianetti4@weebly.com</td>\n",
       "      <td>kabercromby5@squidoo.com</td>\n",
       "      <td>fvolkers6@jugem.jp</td>\n",
       "      <td>kflisher7@list-manage.com</td>\n",
       "      <td>cholttom8@msn.com</td>\n",
       "      <td>cadacot9@dyndns.org</td>\n",
       "      <td>...</td>\n",
       "      <td>gfawdryrj@kickstarter.com</td>\n",
       "      <td>gguichardrk@oakley.com</td>\n",
       "      <td>kmctrustramrl@smugmug.com</td>\n",
       "      <td>mmarchentrm@twitpic.com</td>\n",
       "      <td>raberkirdorn@loc.gov</td>\n",
       "      <td>wbithellro@mediafire.com</td>\n",
       "      <td>jlaurierp@1und1.de</td>\n",
       "      <td>bshadrackrq@issuu.com</td>\n",
       "      <td>jpretiousrr@scribd.com</td>\n",
       "      <td>email</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gender</th>\n",
       "      <td>Female</td>\n",
       "      <td>Male</td>\n",
       "      <td>Female</td>\n",
       "      <td>Female</td>\n",
       "      <td>Male</td>\n",
       "      <td>Female</td>\n",
       "      <td>Female</td>\n",
       "      <td>Female</td>\n",
       "      <td>Female</td>\n",
       "      <td>Male</td>\n",
       "      <td>...</td>\n",
       "      <td>Male</td>\n",
       "      <td>Female</td>\n",
       "      <td>Male</td>\n",
       "      <td>Female</td>\n",
       "      <td>Female</td>\n",
       "      <td>Male</td>\n",
       "      <td>Female</td>\n",
       "      <td>Female</td>\n",
       "      <td>Male</td>\n",
       "      <td>gender</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ip_address</th>\n",
       "      <td>107.233.251.179</td>\n",
       "      <td>150.174.167.55</td>\n",
       "      <td>140.184.222.179</td>\n",
       "      <td>75.169.255.200</td>\n",
       "      <td>222.183.109.137</td>\n",
       "      <td>234.67.16.236</td>\n",
       "      <td>102.246.26.217</td>\n",
       "      <td>247.72.238.49</td>\n",
       "      <td>213.218.116.96</td>\n",
       "      <td>132.14.208.121</td>\n",
       "      <td>...</td>\n",
       "      <td>159.240.21.93</td>\n",
       "      <td>212.139.132.144</td>\n",
       "      <td>240.52.253.96</td>\n",
       "      <td>213.199.184.129</td>\n",
       "      <td>53.114.178.100</td>\n",
       "      <td>165.156.92.220</td>\n",
       "      <td>227.166.153.18</td>\n",
       "      <td>83.220.101.63</td>\n",
       "      <td>36.224.203.141</td>\n",
       "      <td>ip_address</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>phone_number</th>\n",
       "      <td>364-613-4322</td>\n",
       "      <td>613-810-5889</td>\n",
       "      <td>158-740-4562</td>\n",
       "      <td>472-249-0728</td>\n",
       "      <td>657-609-7297</td>\n",
       "      <td>552-683-0589</td>\n",
       "      <td>843-241-0936</td>\n",
       "      <td>725-953-9021</td>\n",
       "      <td>937-620-9140</td>\n",
       "      <td>622-126-0293</td>\n",
       "      <td>...</td>\n",
       "      <td>244-995-8457</td>\n",
       "      <td>691-992-1934</td>\n",
       "      <td>876-708-1212</td>\n",
       "      <td>926-325-4287</td>\n",
       "      <td>780-657-3624</td>\n",
       "      <td>252-569-9522</td>\n",
       "      <td>654-395-7627</td>\n",
       "      <td>785-969-9506</td>\n",
       "      <td>938-898-2068</td>\n",
       "      <td>phone_number</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>home_address</th>\n",
       "      <td>593 Dryden Park</td>\n",
       "      <td>842 Westport Trail</td>\n",
       "      <td>8 Ridgeview Terrace</td>\n",
       "      <td>4890 Acker Lane</td>\n",
       "      <td>12827 Barnett Plaza</td>\n",
       "      <td>16 Bellgrove Pass</td>\n",
       "      <td>10 Grim Hill</td>\n",
       "      <td>5928 Welch Lane</td>\n",
       "      <td>27 Atwood Parkway</td>\n",
       "      <td>42914 Huxley Drive</td>\n",
       "      <td>...</td>\n",
       "      <td>6 Corry Lane</td>\n",
       "      <td>3 Coolidge Point</td>\n",
       "      <td>98011 Buhler Trail</td>\n",
       "      <td>30 Independence Court</td>\n",
       "      <td>500 Roxbury Park</td>\n",
       "      <td>8 Calypso Alley</td>\n",
       "      <td>810 1st Alley</td>\n",
       "      <td>293 5th Way</td>\n",
       "      <td>8 Waubesa Court</td>\n",
       "      <td>home_address</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ssn</th>\n",
       "      <td>875-50-3238</td>\n",
       "      <td>109-37-5461</td>\n",
       "      <td>618-05-4668</td>\n",
       "      <td>677-86-3312</td>\n",
       "      <td>235-54-2795</td>\n",
       "      <td>292-70-7424</td>\n",
       "      <td>508-52-0991</td>\n",
       "      <td>331-74-2147</td>\n",
       "      <td>211-70-1621</td>\n",
       "      <td>104-81-8368</td>\n",
       "      <td>...</td>\n",
       "      <td>364-37-6587</td>\n",
       "      <td>680-21-0177</td>\n",
       "      <td>399-68-5810</td>\n",
       "      <td>502-84-6796</td>\n",
       "      <td>686-30-5830</td>\n",
       "      <td>424-47-7300</td>\n",
       "      <td>236-63-1431</td>\n",
       "      <td>464-91-8544</td>\n",
       "      <td>580-15-5838</td>\n",
       "      <td>ssn</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 2001 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                              0                           1  \\\n",
       "first_name                 Jill                   Frasquito   \n",
       "last_name                  Mish                       Hamer   \n",
       "email         jmish0@sbwire.com  fhamer1@washingtonpost.com   \n",
       "gender                   Female                        Male   \n",
       "ip_address      107.233.251.179              150.174.167.55   \n",
       "phone_number       364-613-4322                613-810-5889   \n",
       "home_address    593 Dryden Park          842 Westport Trail   \n",
       "ssn                 875-50-3238                 109-37-5461   \n",
       "\n",
       "                                      2                        3  \\\n",
       "first_name                       Maddie                    Selia   \n",
       "last_name                    Muggleston                 Wiffield   \n",
       "email         mmuggleston2@amazon.co.jp  swiffield3@edublogs.org   \n",
       "gender                           Female                   Female   \n",
       "ip_address              140.184.222.179           75.169.255.200   \n",
       "phone_number               158-740-4562             472-249-0728   \n",
       "home_address        8 Ridgeview Terrace          4890 Acker Lane   \n",
       "ssn                         618-05-4668              677-86-3312   \n",
       "\n",
       "                                 4                         5  \\\n",
       "first_name                 Raleigh                      Kipp   \n",
       "last_name                  Ianetti                Abercromby   \n",
       "email         rianetti4@weebly.com  kabercromby5@squidoo.com   \n",
       "gender                        Male                    Female   \n",
       "ip_address         222.183.109.137             234.67.16.236   \n",
       "phone_number          657-609-7297              552-683-0589   \n",
       "home_address   12827 Barnett Plaza         16 Bellgrove Pass   \n",
       "ssn                    235-54-2795               292-70-7424   \n",
       "\n",
       "                               6                          7  \\\n",
       "first_name                Flossi                       Kaia   \n",
       "last_name                Volkers                    Flisher   \n",
       "email         fvolkers6@jugem.jp  kflisher7@list-manage.com   \n",
       "gender                    Female                     Female   \n",
       "ip_address        102.246.26.217              247.72.238.49   \n",
       "phone_number        843-241-0936               725-953-9021   \n",
       "home_address        10 Grim Hill            5928 Welch Lane   \n",
       "ssn                  508-52-0991                331-74-2147   \n",
       "\n",
       "                              8                    9      ...       \\\n",
       "first_name             Carolann                  Cam      ...        \n",
       "last_name               Holttom               Adacot      ...        \n",
       "email         cholttom8@msn.com  cadacot9@dyndns.org      ...        \n",
       "gender                   Female                 Male      ...        \n",
       "ip_address       213.218.116.96       132.14.208.121      ...        \n",
       "phone_number       937-620-9140         622-126-0293      ...        \n",
       "home_address  27 Atwood Parkway   42914 Huxley Drive      ...        \n",
       "ssn                 211-70-1621          104-81-8368      ...        \n",
       "\n",
       "                                   1991                    1992  \\\n",
       "first_name                       Gaston                  Gennie   \n",
       "last_name                        Fawdry                Guichard   \n",
       "email         gfawdryrj@kickstarter.com  gguichardrk@oakley.com   \n",
       "gender                             Male                  Female   \n",
       "ip_address                159.240.21.93         212.139.132.144   \n",
       "phone_number               244-995-8457            691-992-1934   \n",
       "home_address               6 Corry Lane        3 Coolidge Point   \n",
       "ssn                         364-37-6587             680-21-0177   \n",
       "\n",
       "                                   1993                     1994  \\\n",
       "first_name                         Keen                  Myranda   \n",
       "last_name                    McTrustram                 Marchent   \n",
       "email         kmctrustramrl@smugmug.com  mmarchentrm@twitpic.com   \n",
       "gender                             Male                   Female   \n",
       "ip_address                240.52.253.96          213.199.184.129   \n",
       "phone_number               876-708-1212             926-325-4287   \n",
       "home_address         98011 Buhler Trail    30 Independence Court   \n",
       "ssn                         399-68-5810              502-84-6796   \n",
       "\n",
       "                              1995                      1996  \\\n",
       "first_name                  Regina                      Wolf   \n",
       "last_name                Aberkirdo                   Bithell   \n",
       "email         raberkirdorn@loc.gov  wbithellro@mediafire.com   \n",
       "gender                      Female                      Male   \n",
       "ip_address          53.114.178.100            165.156.92.220   \n",
       "phone_number          780-657-3624              252-569-9522   \n",
       "home_address      500 Roxbury Park           8 Calypso Alley   \n",
       "ssn                    686-30-5830               424-47-7300   \n",
       "\n",
       "                            1997                   1998  \\\n",
       "first_name                Joanna                 Bianca   \n",
       "last_name                 Laurie               Shadrack   \n",
       "email         jlaurierp@1und1.de  bshadrackrq@issuu.com   \n",
       "gender                    Female                 Female   \n",
       "ip_address        227.166.153.18          83.220.101.63   \n",
       "phone_number        654-395-7627           785-969-9506   \n",
       "home_address       810 1st Alley            293 5th Way   \n",
       "ssn                  236-63-1431            464-91-8544   \n",
       "\n",
       "                                1999      category  \n",
       "first_name                     Jabez    first_name  \n",
       "last_name                   Pretious     last_name  \n",
       "email         jpretiousrr@scribd.com         email  \n",
       "gender                          Male        gender  \n",
       "ip_address            36.224.203.141    ip_address  \n",
       "phone_number            938-898-2068  phone_number  \n",
       "home_address         8 Waubesa Court  home_address  \n",
       "ssn                      580-15-5838           ssn  \n",
       "\n",
       "[8 rows x 2001 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.transpose()\n",
    "df['category'] = df.index\n",
    "df.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After this, we need to melt the dataframe so that it has a long shape instead of a wide shape. By using the 'category' column that we added earlier as the id variable for the melt, we end up with a new dataframe that has the schema ('category', 'variable', 'value'). The 'variable' variable here is useless to us so we can drop it from the dataframe. Now we have a dataframe that has the schema ('category', 'value')."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>category</th>\n",
       "      <th>value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>first_name</td>\n",
       "      <td>Jill</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>last_name</td>\n",
       "      <td>Mish</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>email</td>\n",
       "      <td>jmish0@sbwire.com</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>gender</td>\n",
       "      <td>Female</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ip_address</td>\n",
       "      <td>107.233.251.179</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>phone_number</td>\n",
       "      <td>364-613-4322</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>home_address</td>\n",
       "      <td>593 Dryden Park</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>ssn</td>\n",
       "      <td>875-50-3238</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>first_name</td>\n",
       "      <td>Frasquito</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>last_name</td>\n",
       "      <td>Hamer</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       category              value\n",
       "0    first_name               Jill\n",
       "1     last_name               Mish\n",
       "2         email  jmish0@sbwire.com\n",
       "3        gender             Female\n",
       "4    ip_address    107.233.251.179\n",
       "5  phone_number       364-613-4322\n",
       "6  home_address    593 Dryden Park\n",
       "7           ssn        875-50-3238\n",
       "8    first_name          Frasquito\n",
       "9     last_name              Hamer"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.melt(id_vars=['category'])\n",
    "df = df.drop('variable', axis=1)\n",
    "df.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using this new dataframe, we have extactly what we need to train our model, but we still need to check for missing/NaN values in the dataset.\n",
    "\n",
    "<h2><center>ADD SOME CODE FOR DATA CLEANING HERE </center></h2>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Building the Model #"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After cleaning and reshaping our training dataset, we can start building our model. Lets take a quick look at our data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>category</th>\n",
       "      <th>value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>first_name</td>\n",
       "      <td>Jill</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>last_name</td>\n",
       "      <td>Mish</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>email</td>\n",
       "      <td>jmish0@sbwire.com</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>gender</td>\n",
       "      <td>Female</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ip_address</td>\n",
       "      <td>107.233.251.179</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>phone_number</td>\n",
       "      <td>364-613-4322</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>home_address</td>\n",
       "      <td>593 Dryden Park</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>ssn</td>\n",
       "      <td>875-50-3238</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>first_name</td>\n",
       "      <td>Frasquito</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>last_name</td>\n",
       "      <td>Hamer</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       category              value\n",
       "0    first_name               Jill\n",
       "1     last_name               Mish\n",
       "2         email  jmish0@sbwire.com\n",
       "3        gender             Female\n",
       "4    ip_address    107.233.251.179\n",
       "5  phone_number       364-613-4322\n",
       "6  home_address    593 Dryden Park\n",
       "7           ssn        875-50-3238\n",
       "8    first_name          Frasquito\n",
       "9     last_name              Hamer"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To build our model, we can't use strings which is all we have in our dataset. So we need to convert these strings into some sort of number that the model will be able to do math with.\n",
    "\n",
    "Enter scikit-learn's HashingVectorizer. This vectorizer performs a hashing algorithm on the data you pass into it and gives us a sparse matrix of floats that describe each string. By default, the HashingVectorizer does its vectorization assuming there are $ 2^{20} $ features. According to the documentation this is usually a good number of features for text classification."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['vectorizer.joblib']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hash_vect = HashingVectorizer()\n",
    "jl.dump(hash_vect, 'vectorizer.joblib')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The next step is to extract our dependent and independent variables from the dataframe. Our independent variable(X) will be the 'value' column of our dataframe and our dependent variable(y) will be the 'category' column of the dataframe. However, since our independent variable has to be numerical, we need to use our vectorizer to vectorize the strings."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['value'] = df['value'].astype(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = hash_vect.fit_transform(df['value'])\n",
    "y = df['category']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After extracting our variables we can create a test/train split to build our model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Based on scikit-learn's machine learning cheat sheet, we decided that a LinearSVC would most likely be the best model for this problem.\n",
    "\n",
    "So now we can create the model and train it using our training data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8484848484848485"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf = LinearSVC()\n",
    "clf.fit(X_train, y_train)\n",
    "clf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['model.joblib']"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jl.dump(clf, 'model.joblib')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here we can see that our model performs pretty well out of the box, with an accuracy around 75%. However, we haven't done any hyperparameterization yet. In later sprints, we will refine this model using cross-validation/grid search/etc."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Building Classification Objects ##"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "When we are given new data, we will perform the same sort of data massaging as we did when building our model. However, we will use the model we built earlier to create a new column in the massaged dataframe. This column will include the 'guess' that our model made for that value."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "File b'test_data.csv' does not exist",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-31-f71dfc1a5e61>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'test_data.csv'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mdf_orig\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtranspose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'category'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdrop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'id'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmelt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mid_vars\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'category'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdrop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'variable'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36mparser_f\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, escapechar, comment, encoding, dialect, tupleize_cols, error_bad_lines, warn_bad_lines, skipfooter, doublequote, delim_whitespace, low_memory, memory_map, float_precision)\u001b[0m\n\u001b[1;32m    676\u001b[0m                     skip_blank_lines=skip_blank_lines)\n\u001b[1;32m    677\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 678\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    679\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    680\u001b[0m     \u001b[0mparser_f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m    438\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    439\u001b[0m     \u001b[0;31m# Create the parser.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 440\u001b[0;31m     \u001b[0mparser\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTextFileReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    441\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    442\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mchunksize\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0miterator\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[1;32m    785\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'has_index_names'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'has_index_names'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    786\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 787\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    788\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    789\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_make_engine\u001b[0;34m(self, engine)\u001b[0m\n\u001b[1;32m   1012\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mengine\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'c'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1013\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'c'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1014\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCParserWrapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1015\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1016\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'python'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, src, **kwds)\u001b[0m\n\u001b[1;32m   1706\u001b[0m         \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'usecols'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0musecols\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1707\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1708\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparsers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTextReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1709\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1710\u001b[0m         \u001b[0mpassed_names\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnames\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader.__cinit__\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._setup_parser_source\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: File b'test_data.csv' does not exist"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv('test_data.csv')\n",
    "df_orig = df\n",
    "df = df.transpose()\n",
    "df['category'] = df.index\n",
    "df = df.drop('id').melt(id_vars=['category']).drop('variable', axis=1)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h4>NOTE: This dataset has different column names for fname and lname</h4>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['guess'] = clf.predict(hash_vect.fit_transform(df['value']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>category</th>\n",
       "      <th>value</th>\n",
       "      <th>guess</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>firstname</td>\n",
       "      <td>Giacomo</td>\n",
       "      <td>lname</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>lastname</td>\n",
       "      <td>Pulford</td>\n",
       "      <td>lname</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>email</td>\n",
       "      <td>gpulford0@wsj.com</td>\n",
       "      <td>email</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>gender</td>\n",
       "      <td>Male</td>\n",
       "      <td>gender</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ip_address</td>\n",
       "      <td>74.141.110.4</td>\n",
       "      <td>ip_address</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>phone</td>\n",
       "      <td>934-562-0496</td>\n",
       "      <td>phone</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>address</td>\n",
       "      <td>6144 Bonner Pass</td>\n",
       "      <td>address</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>ssn</td>\n",
       "      <td>446-52-7673</td>\n",
       "      <td>money</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>firstname</td>\n",
       "      <td>Virge</td>\n",
       "      <td>fname</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>lastname</td>\n",
       "      <td>Le Grove</td>\n",
       "      <td>lname</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     category              value       guess\n",
       "0   firstname            Giacomo       lname\n",
       "1    lastname            Pulford       lname\n",
       "2       email  gpulford0@wsj.com       email\n",
       "3      gender               Male      gender\n",
       "4  ip_address       74.141.110.4  ip_address\n",
       "5       phone       934-562-0496       phone\n",
       "6     address   6144 Bonner Pass     address\n",
       "7         ssn        446-52-7673       money\n",
       "8   firstname              Virge       fname\n",
       "9    lastname           Le Grove       lname"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using this new column, we will need to calculate some sort of 'precision' score for each 'category' in the original dataset. Using this precision score, we can determine whether a 'category' belongs to one of the categories that our model can predict. If a category's 'precision' score is too low, then we can consider it as being some sort of new category of data that we haven't encountered yet."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using this new data frame we can group the guesses together to present our findings to the user."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>category</th>\n",
       "      <th>value</th>\n",
       "      <th>guess</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>firstname</td>\n",
       "      <td>Giacomo</td>\n",
       "      <td>lname</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>lastname</td>\n",
       "      <td>Pulford</td>\n",
       "      <td>lname</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>email</td>\n",
       "      <td>gpulford0@wsj.com</td>\n",
       "      <td>email</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>gender</td>\n",
       "      <td>Male</td>\n",
       "      <td>gender</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ip_address</td>\n",
       "      <td>74.141.110.4</td>\n",
       "      <td>ip_address</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     category              value       guess\n",
       "0   firstname            Giacomo       lname\n",
       "1    lastname            Pulford       lname\n",
       "2       email  gpulford0@wsj.com       email\n",
       "3      gender               Male      gender\n",
       "4  ip_address       74.141.110.4  ip_address"
      ]
     },
     "execution_count": 156,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We want to build a frequency matrix for "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'firstname': {'lname': 853, 'fname': 143, 'address': 1, 'credit_card': 3},\n",
       " 'lastname': {'lname': 989,\n",
       "  'fname': 5,\n",
       "  'email': 4,\n",
       "  'ssn': 1,\n",
       "  'credit_card': 1},\n",
       " 'email': {'email': 1000},\n",
       " 'gender': {'gender': 1000},\n",
       " 'ip_address': {'ip_address': 956, 'money': 44},\n",
       " 'phone': {'phone': 838, 'ssn': 129, 'ip_address': 26, 'lname': 7},\n",
       " 'address': {'address': 1000},\n",
       " 'ssn': {'money': 396,\n",
       "  'ssn': 433,\n",
       "  'ip_address': 143,\n",
       "  'phone': 24,\n",
       "  'fname': 1,\n",
       "  'lname': 2,\n",
       "  'email': 1}}"
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result = {}\n",
    "for t in df.itertuples():\n",
    "    actual = t.category\n",
    "    guess = t.guess\n",
    "    \n",
    "    if actual not in result.keys():\n",
    "        result[actual] = {}\n",
    "        \n",
    "    if guess not in result[actual].keys():\n",
    "        result[actual][guess] = 1\n",
    "    else:\n",
    "        result[actual][guess] = result[actual][guess] + 1\n",
    "\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "res_df=pd.DataFrame(result).fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "firstname      853.0\n",
       "lastname       989.0\n",
       "email         1000.0\n",
       "gender        1000.0\n",
       "ip_address     956.0\n",
       "phone          838.0\n",
       "address       1000.0\n",
       "ssn            433.0\n",
       "dtype: float64"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res_df.apply(max)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>firstname</th>\n",
       "      <th>lastname</th>\n",
       "      <th>email</th>\n",
       "      <th>gender</th>\n",
       "      <th>ip_address</th>\n",
       "      <th>phone</th>\n",
       "      <th>address</th>\n",
       "      <th>ssn</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>address</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1000.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>credit_card</th>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>email</th>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fname</th>\n",
       "      <td>143.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gender</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ip_address</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>956.0</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>143.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lname</th>\n",
       "      <td>853.0</td>\n",
       "      <td>989.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>money</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>44.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>396.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>phone</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>838.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ssn</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>129.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>433.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             firstname  lastname   email  gender  ip_address  phone  address  \\\n",
       "address            1.0       0.0     0.0     0.0         0.0    0.0   1000.0   \n",
       "credit_card        3.0       1.0     0.0     0.0         0.0    0.0      0.0   \n",
       "email              0.0       4.0  1000.0     0.0         0.0    0.0      0.0   \n",
       "fname            143.0       5.0     0.0     0.0         0.0    0.0      0.0   \n",
       "gender             0.0       0.0     0.0  1000.0         0.0    0.0      0.0   \n",
       "ip_address         0.0       0.0     0.0     0.0       956.0   26.0      0.0   \n",
       "lname            853.0     989.0     0.0     0.0         0.0    7.0      0.0   \n",
       "money              0.0       0.0     0.0     0.0        44.0    0.0      0.0   \n",
       "phone              0.0       0.0     0.0     0.0         0.0  838.0      0.0   \n",
       "ssn                0.0       1.0     0.0     0.0         0.0  129.0      0.0   \n",
       "\n",
       "               ssn  \n",
       "address        0.0  \n",
       "credit_card    0.0  \n",
       "email          1.0  \n",
       "fname          1.0  \n",
       "gender         0.0  \n",
       "ip_address   143.0  \n",
       "lname          2.0  \n",
       "money        396.0  \n",
       "phone         24.0  \n",
       "ssn          433.0  "
      ]
     },
     "execution_count": 160,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "firstname     0.853\n",
       "lastname      0.989\n",
       "email         1.000\n",
       "gender        1.000\n",
       "ip_address    0.956\n",
       "phone         0.838\n",
       "address       1.000\n",
       "ssn           0.433\n",
       "dtype: float64"
      ]
     },
     "execution_count": 161,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "freq = res_df.apply(max)/res_df.apply(sum)\n",
    "freq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "    \"name\": \"lname\",\n",
      "    \"columns\": [\"firstname\"],\n",
      "    \"examples\": [\"Giacomo\", \"Virge\", \"Fielding\", \"Zebadiah\", \"Chrotoem\"]\n",
      "}\n",
      "\n",
      "            \n",
      "{\n",
      "    \"name\": \"email\",\n",
      "    \"columns\": [\"email\"],\n",
      "    \"examples\": [\"gpulford0@wsj.com\", \"vlegrove1@reference.com\", \"fbaiyle2@imageshack.us\", \"zrichardot3@spotify.com\", \"coleszkiewicz4@vkontakte.ru\"]\n",
      "}\n",
      "\n",
      "            \n",
      "{\n",
      "    \"name\": \"gender\",\n",
      "    \"columns\": [\"gender\"],\n",
      "    \"examples\": [\"Male\", \"Male\", \"Male\", \"Male\", \"Male\"]\n",
      "}\n",
      "\n",
      "            \n",
      "{\n",
      "    \"name\": \"ip_address\",\n",
      "    \"columns\": [\"ip_address\"],\n",
      "    \"examples\": [\"74.141.110.4\", \"220.235.6.159\", \"222.205.231.178\", \"9.90.228.199\", \"35.76.198.142\"]\n",
      "}\n",
      "\n",
      "            \n",
      "{\n",
      "    \"name\": \"phone\",\n",
      "    \"columns\": [\"phone\"],\n",
      "    \"examples\": [\"934-562-0496\", \"736-696-7582\", \"149-779-8128\", \"383-641-4571\", \"215-835-7270\"]\n",
      "}\n",
      "\n",
      "            \n",
      "{\n",
      "    \"name\": \"address\",\n",
      "    \"columns\": [\"address\"],\n",
      "    \"examples\": [\"6144 Bonner Pass\", \"223 Loftsgordon Plaza\", \"483 Oneill Place\", \"99485 Paget Parkway\", \"2142 Anzinger Plaza\"]\n",
      "}\n",
      "\n",
      "            \n",
      "{\n",
      "    \"name\": \"ssn\",\n",
      "    \"columns\": [\"ssn\"],\n",
      "    \"examples\": [\"446-52-7673\", \"803-76-8610\", \"210-72-5738\", \"671-38-8124\", \"604-13-4970\"]\n",
      "}\n",
      "\n",
      "            \n",
      "{\n",
      "    \"name\": \"money\",\n",
      "    \"columns\": [\"ssn\"],\n",
      "    \"examples\": [\"446-52-7673\", \"803-76-8610\", \"210-72-5738\", \"671-38-8124\", \"604-13-4970\"]\n",
      "}\n",
      "\n",
      "            \n"
     ]
    }
   ],
   "source": [
    "class Classification:\n",
    "\n",
    "    def add_column(self, column):\n",
    "        self.columns.append(column)\n",
    "\n",
    "    def add_example(self, example):\n",
    "        self.examples.append(example)\n",
    "\n",
    "    def to_json(self):\n",
    "        return f\"\"\"\\\n",
    "{{\n",
    "    \"name\": \"{self.name}\",\n",
    "    \"columns\": {self.columns},\n",
    "    \"examples\": {self.examples}\n",
    "}}\n",
    "\n",
    "            \"\"\".replace(\"'\",'\"')\n",
    "    \n",
    "    def __eq__(self, other):\n",
    "        return isinstance(other, self.__class__) and self.name == other.name\n",
    "    \n",
    "    def __str__(self):\n",
    "        return f\"\"\"Name: {self.name}\n",
    "            Columns: {self.columns}\n",
    "            Examples: {self.examples}\n",
    "        \"\"\"\n",
    "        \n",
    "    def __init__(self, name):\n",
    "        self.name = name\n",
    "        self.examples = []\n",
    "        self.columns = []\n",
    "        \n",
    "classification_map = {}\n",
    "for k,v in freq.items():\n",
    "    if v >= 0.75:\n",
    "        classification_map[k] = [res_df[k].idxmax()]\n",
    "    else:\n",
    "        classification_map[k] = [res_df[k].idxmax(), res_df[k].drop(res_df[k].idxmax()).idxmax()]\n",
    "\n",
    "classifications = []\n",
    "for k, v in classification_map.items():\n",
    "    for c in v:\n",
    "        classification = Classification(c)\n",
    "        if classification not in classifications:\n",
    "            classification.add_column(k)\n",
    "            for ex in df[df['category'] == k]['value'].head():\n",
    "                classification.add_example(ex)\n",
    "            classifications.append(classification)\n",
    "for cls in classifications:\n",
    "    print(cls.to_json())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}