SchoolBudgetData.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Labeling school budget data\n",
    "\n",
    "Goal: Perform NLP on school budget data to predict accurate labels for the expenditure category using multiclass classification models from scikit learn. \n",
    "\n",
    "We will build a human-in-the-loop system by assigning a probability of labels to each line item.  \n",
    "\n",
    "Author: Sushant N. More\n",
    "\n",
    "Data from [DrivenData](https://www.drivendata.org/competitions/46/box-plots-for-education-reboot/data/) \n",
    "\n",
    "Ref: Datacamp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.preprocessing import FunctionTransformer\n",
    "from sklearn.pipeline import FeatureUnion\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import Imputer\n",
    "from sklearn.ensemble import RandomForestClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_train = pd.read_csv('TrainingData.csv', index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Function</th>\n",
       "      <th>Use</th>\n",
       "      <th>Sharing</th>\n",
       "      <th>Reporting</th>\n",
       "      <th>Student_Type</th>\n",
       "      <th>Position_Type</th>\n",
       "      <th>Object_Type</th>\n",
       "      <th>Pre_K</th>\n",
       "      <th>Operating_Status</th>\n",
       "      <th>Object_Description</th>\n",
       "      <th>...</th>\n",
       "      <th>Sub_Object_Description</th>\n",
       "      <th>Location_Description</th>\n",
       "      <th>FTE</th>\n",
       "      <th>Function_Description</th>\n",
       "      <th>Facility_or_Department</th>\n",
       "      <th>Position_Extra</th>\n",
       "      <th>Total</th>\n",
       "      <th>Program_Description</th>\n",
       "      <th>Fund_Description</th>\n",
       "      <th>Text_1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>134338</th>\n",
       "      <td>Teacher Compensation</td>\n",
       "      <td>Instruction</td>\n",
       "      <td>School Reported</td>\n",
       "      <td>School</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>Teacher</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>PreK-12 Operating</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>KINDERGARTEN</td>\n",
       "      <td>50471.810</td>\n",
       "      <td>KINDERGARTEN</td>\n",
       "      <td>General Fund</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>206341</th>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>Non-Operating</td>\n",
       "      <td>CONTRACTOR SERVICES</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>RGN  GOB</td>\n",
       "      <td>NaN</td>\n",
       "      <td>UNDESIGNATED</td>\n",
       "      <td>3477.860</td>\n",
       "      <td>BUILDING IMPROVEMENT SERVICES</td>\n",
       "      <td>NaN</td>\n",
       "      <td>BUILDING IMPROVEMENT SERVICES</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>326408</th>\n",
       "      <td>Teacher Compensation</td>\n",
       "      <td>Instruction</td>\n",
       "      <td>School Reported</td>\n",
       "      <td>School</td>\n",
       "      <td>Unspecified</td>\n",
       "      <td>Teacher</td>\n",
       "      <td>Base Salary/Compensation</td>\n",
       "      <td>Non PreK</td>\n",
       "      <td>PreK-12 Operating</td>\n",
       "      <td>Personal Services - Teachers</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>TEACHER</td>\n",
       "      <td>62237.130</td>\n",
       "      <td>Instruction - Regular</td>\n",
       "      <td>General Purpose School</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>364634</th>\n",
       "      <td>Substitute Compensation</td>\n",
       "      <td>Instruction</td>\n",
       "      <td>School Reported</td>\n",
       "      <td>School</td>\n",
       "      <td>Unspecified</td>\n",
       "      <td>Substitute</td>\n",
       "      <td>Benefits</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>PreK-12 Operating</td>\n",
       "      <td>EMPLOYEE BENEFITS</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>UNALLOC BUDGETS/SCHOOLS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>PROFESSIONAL-INSTRUCTIONAL</td>\n",
       "      <td>22.300</td>\n",
       "      <td>GENERAL MIDDLE/JUNIOR HIGH SCH</td>\n",
       "      <td>NaN</td>\n",
       "      <td>REGULAR INSTRUCTION</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47683</th>\n",
       "      <td>Substitute Compensation</td>\n",
       "      <td>Instruction</td>\n",
       "      <td>School Reported</td>\n",
       "      <td>School</td>\n",
       "      <td>Unspecified</td>\n",
       "      <td>Teacher</td>\n",
       "      <td>Substitute Compensation</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>PreK-12 Operating</td>\n",
       "      <td>TEACHER COVERAGE FOR TEACHER</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NON-PROJECT</td>\n",
       "      <td>NaN</td>\n",
       "      <td>PROFESSIONAL-INSTRUCTIONAL</td>\n",
       "      <td>54.166</td>\n",
       "      <td>GENERAL HIGH SCHOOL EDUCATION</td>\n",
       "      <td>NaN</td>\n",
       "      <td>REGULAR INSTRUCTION</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                       Function          Use          Sharing Reporting  \\\n",
       "134338     Teacher Compensation  Instruction  School Reported    School   \n",
       "206341                 NO_LABEL     NO_LABEL         NO_LABEL  NO_LABEL   \n",
       "326408     Teacher Compensation  Instruction  School Reported    School   \n",
       "364634  Substitute Compensation  Instruction  School Reported    School   \n",
       "47683   Substitute Compensation  Instruction  School Reported    School   \n",
       "\n",
       "       Student_Type Position_Type               Object_Type     Pre_K  \\\n",
       "134338     NO_LABEL       Teacher                  NO_LABEL  NO_LABEL   \n",
       "206341     NO_LABEL      NO_LABEL                  NO_LABEL  NO_LABEL   \n",
       "326408  Unspecified       Teacher  Base Salary/Compensation  Non PreK   \n",
       "364634  Unspecified    Substitute                  Benefits  NO_LABEL   \n",
       "47683   Unspecified       Teacher   Substitute Compensation  NO_LABEL   \n",
       "\n",
       "         Operating_Status            Object_Description  \\\n",
       "134338  PreK-12 Operating                           NaN   \n",
       "206341      Non-Operating           CONTRACTOR SERVICES   \n",
       "326408  PreK-12 Operating  Personal Services - Teachers   \n",
       "364634  PreK-12 Operating             EMPLOYEE BENEFITS   \n",
       "47683   PreK-12 Operating  TEACHER COVERAGE FOR TEACHER   \n",
       "\n",
       "                    ...               Sub_Object_Description  \\\n",
       "134338              ...                                  NaN   \n",
       "206341              ...                                  NaN   \n",
       "326408              ...                                  NaN   \n",
       "364634              ...                                  NaN   \n",
       "47683               ...                                  NaN   \n",
       "\n",
       "       Location_Description  FTE     Function_Description  \\\n",
       "134338                  NaN  1.0                      NaN   \n",
       "206341                  NaN  NaN                 RGN  GOB   \n",
       "326408                  NaN  1.0                      NaN   \n",
       "364634                  NaN  NaN  UNALLOC BUDGETS/SCHOOLS   \n",
       "47683                   NaN  NaN              NON-PROJECT   \n",
       "\n",
       "       Facility_or_Department              Position_Extra      Total  \\\n",
       "134338                    NaN               KINDERGARTEN   50471.810   \n",
       "206341                    NaN                UNDESIGNATED   3477.860   \n",
       "326408                    NaN                     TEACHER  62237.130   \n",
       "364634                    NaN  PROFESSIONAL-INSTRUCTIONAL     22.300   \n",
       "47683                     NaN  PROFESSIONAL-INSTRUCTIONAL     54.166   \n",
       "\n",
       "                   Program_Description        Fund_Description  \\\n",
       "134338                    KINDERGARTEN            General Fund   \n",
       "206341   BUILDING IMPROVEMENT SERVICES                     NaN   \n",
       "326408           Instruction - Regular  General Purpose School   \n",
       "364634  GENERAL MIDDLE/JUNIOR HIGH SCH                     NaN   \n",
       "47683    GENERAL HIGH SCHOOL EDUCATION                     NaN   \n",
       "\n",
       "                               Text_1  \n",
       "134338                            NaN  \n",
       "206341  BUILDING IMPROVEMENT SERVICES  \n",
       "326408                            NaN  \n",
       "364634            REGULAR INSTRUCTION  \n",
       "47683             REGULAR INSTRUCTION  \n",
       "\n",
       "[5 rows x 25 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 400277 entries, 134338 to 415831\n",
      "Data columns (total 25 columns):\n",
      "Function                  400277 non-null object\n",
      "Use                       400277 non-null object\n",
      "Sharing                   400277 non-null object\n",
      "Reporting                 400277 non-null object\n",
      "Student_Type              400277 non-null object\n",
      "Position_Type             400277 non-null object\n",
      "Object_Type               400277 non-null object\n",
      "Pre_K                     400277 non-null object\n",
      "Operating_Status          400277 non-null object\n",
      "Object_Description        375493 non-null object\n",
      "Text_2                    88217 non-null object\n",
      "SubFund_Description       306855 non-null object\n",
      "Job_Title_Description     292743 non-null object\n",
      "Text_3                    179964 non-null object\n",
      "Text_4                    53746 non-null object\n",
      "Sub_Object_Description    91603 non-null object\n",
      "Location_Description      162054 non-null object\n",
      "FTE                       126071 non-null float64\n",
      "Function_Description      342195 non-null object\n",
      "Facility_or_Department    53886 non-null object\n",
      "Position_Extra            264764 non-null object\n",
      "Total                     395722 non-null float64\n",
      "Program_Description       304660 non-null object\n",
      "Fund_Description          202877 non-null object\n",
      "Text_1                    292285 non-null object\n",
      "dtypes: float64(2), object(23)\n",
      "memory usage: 79.4+ MB\n"
     ]
    }
   ],
   "source": [
    "df_train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Function</th>\n",
       "      <th>Use</th>\n",
       "      <th>Sharing</th>\n",
       "      <th>Reporting</th>\n",
       "      <th>Student_Type</th>\n",
       "      <th>Position_Type</th>\n",
       "      <th>Object_Type</th>\n",
       "      <th>Pre_K</th>\n",
       "      <th>Operating_Status</th>\n",
       "      <th>Object_Description</th>\n",
       "      <th>...</th>\n",
       "      <th>Sub_Object_Description</th>\n",
       "      <th>Location_Description</th>\n",
       "      <th>FTE</th>\n",
       "      <th>Function_Description</th>\n",
       "      <th>Facility_or_Department</th>\n",
       "      <th>Position_Extra</th>\n",
       "      <th>Total</th>\n",
       "      <th>Program_Description</th>\n",
       "      <th>Fund_Description</th>\n",
       "      <th>Text_1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>109283</th>\n",
       "      <td>Professional Development</td>\n",
       "      <td>ISPD</td>\n",
       "      <td>Shared Services</td>\n",
       "      <td>Non-School</td>\n",
       "      <td>Unspecified</td>\n",
       "      <td>Instructional Coach</td>\n",
       "      <td>Other Compensation/Stipend</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>PreK-12 Operating</td>\n",
       "      <td>WORKSHOP PARTICIPANT</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>STAFF DEV AND INSTR MEDIA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>INST STAFF TRAINING SVCS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>48.620000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GENERAL FUND</td>\n",
       "      <td>STAFF DEV AND INSTR MEDIA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102430</th>\n",
       "      <td>Substitute Compensation</td>\n",
       "      <td>Instruction</td>\n",
       "      <td>School Reported</td>\n",
       "      <td>School</td>\n",
       "      <td>Unspecified</td>\n",
       "      <td>Substitute</td>\n",
       "      <td>Base Salary/Compensation</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>PreK-12 Operating</td>\n",
       "      <td>SALARIES OF PART TIME EMPLOYEE</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.00431</td>\n",
       "      <td>TITLE II,D</td>\n",
       "      <td>NaN</td>\n",
       "      <td>PROFESSIONAL-INSTRUCTIONAL</td>\n",
       "      <td>128.824985</td>\n",
       "      <td>INSTRUCTIONAL STAFF TRAINING</td>\n",
       "      <td>NaN</td>\n",
       "      <td>INSTRUCTIONAL STAFF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>413949</th>\n",
       "      <td>Parent &amp; Community Relations</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>School Reported</td>\n",
       "      <td>School</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>Other</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>PreK-12 Operating</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.00000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>PARENT/TITLE I</td>\n",
       "      <td>4902.290000</td>\n",
       "      <td>Misc</td>\n",
       "      <td>Schoolwide Schools</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>433672</th>\n",
       "      <td>Library &amp; Media</td>\n",
       "      <td>Instruction</td>\n",
       "      <td>School on Central Budgets</td>\n",
       "      <td>Non-School</td>\n",
       "      <td>Unspecified</td>\n",
       "      <td>Librarian</td>\n",
       "      <td>Benefits</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>PreK-12 Operating</td>\n",
       "      <td>EMPLOYEE BENEFITS</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ED RESOURCE SERVICES</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NON-PROJECT</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OFFICE/ADMINISTRATIVE SUPPORT</td>\n",
       "      <td>4020.290000</td>\n",
       "      <td>MEDIA SUPPORT SERVICES</td>\n",
       "      <td>NaN</td>\n",
       "      <td>INSTRUCTIONAL STAFF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415831</th>\n",
       "      <td>Substitute Compensation</td>\n",
       "      <td>Instruction</td>\n",
       "      <td>School Reported</td>\n",
       "      <td>School</td>\n",
       "      <td>Poverty</td>\n",
       "      <td>Substitute</td>\n",
       "      <td>Substitute Compensation</td>\n",
       "      <td>Non PreK</td>\n",
       "      <td>PreK-12 Operating</td>\n",
       "      <td>Salaries And Wages For Substitute Professionals</td>\n",
       "      <td>...</td>\n",
       "      <td>Inservice Substitute Teachers Grant Funded</td>\n",
       "      <td>School</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Instruction</td>\n",
       "      <td>Instruction And Curriculum</td>\n",
       "      <td>CERTIFIED SUBSTITUTE</td>\n",
       "      <td>46.530000</td>\n",
       "      <td>Accelerated Education</td>\n",
       "      <td>\"Title  Part A Improving Basic Programs\"</td>\n",
       "      <td>MISCELLANEOUS</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                            Function          Use                    Sharing  \\\n",
       "109283      Professional Development         ISPD            Shared Services   \n",
       "102430       Substitute Compensation  Instruction            School Reported   \n",
       "413949  Parent & Community Relations     NO_LABEL            School Reported   \n",
       "433672               Library & Media  Instruction  School on Central Budgets   \n",
       "415831       Substitute Compensation  Instruction            School Reported   \n",
       "\n",
       "         Reporting Student_Type        Position_Type  \\\n",
       "109283  Non-School  Unspecified  Instructional Coach   \n",
       "102430      School  Unspecified           Substitute   \n",
       "413949      School     NO_LABEL                Other   \n",
       "433672  Non-School  Unspecified            Librarian   \n",
       "415831      School      Poverty           Substitute   \n",
       "\n",
       "                       Object_Type     Pre_K   Operating_Status  \\\n",
       "109283  Other Compensation/Stipend  NO_LABEL  PreK-12 Operating   \n",
       "102430    Base Salary/Compensation  NO_LABEL  PreK-12 Operating   \n",
       "413949                    NO_LABEL  NO_LABEL  PreK-12 Operating   \n",
       "433672                    Benefits  NO_LABEL  PreK-12 Operating   \n",
       "415831     Substitute Compensation  Non PreK  PreK-12 Operating   \n",
       "\n",
       "                                     Object_Description  \\\n",
       "109283                   WORKSHOP PARTICIPANT             \n",
       "102430                   SALARIES OF PART TIME EMPLOYEE   \n",
       "413949                                              NaN   \n",
       "433672                                EMPLOYEE BENEFITS   \n",
       "415831  Salaries And Wages For Substitute Professionals   \n",
       "\n",
       "                     ...                \\\n",
       "109283               ...                 \n",
       "102430               ...                 \n",
       "413949               ...                 \n",
       "433672               ...                 \n",
       "415831               ...                 \n",
       "\n",
       "                            Sub_Object_Description  \\\n",
       "109283                                         NaN   \n",
       "102430                                         NaN   \n",
       "413949                                         NaN   \n",
       "433672                                         NaN   \n",
       "415831  Inservice Substitute Teachers Grant Funded   \n",
       "\n",
       "                  Location_Description      FTE  \\\n",
       "109283  STAFF DEV AND INSTR MEDIA           NaN   \n",
       "102430                             NaN  0.00431   \n",
       "413949                             NaN  1.00000   \n",
       "433672            ED RESOURCE SERVICES      NaN   \n",
       "415831                         School       NaN   \n",
       "\n",
       "                  Function_Description      Facility_or_Department  \\\n",
       "109283  INST STAFF TRAINING SVCS                               NaN   \n",
       "102430                      TITLE II,D                         NaN   \n",
       "413949                             NaN                         NaN   \n",
       "433672                     NON-PROJECT                         NaN   \n",
       "415831                     Instruction  Instruction And Curriculum   \n",
       "\n",
       "                       Position_Extra        Total  \\\n",
       "109283                            NaN    48.620000   \n",
       "102430     PROFESSIONAL-INSTRUCTIONAL   128.824985   \n",
       "413949                 PARENT/TITLE I  4902.290000   \n",
       "433672  OFFICE/ADMINISTRATIVE SUPPORT  4020.290000   \n",
       "415831           CERTIFIED SUBSTITUTE    46.530000   \n",
       "\n",
       "                 Program_Description  \\\n",
       "109283                           NaN   \n",
       "102430  INSTRUCTIONAL STAFF TRAINING   \n",
       "413949                          Misc   \n",
       "433672        MEDIA SUPPORT SERVICES   \n",
       "415831         Accelerated Education   \n",
       "\n",
       "                                Fund_Description  \\\n",
       "109283            GENERAL FUND                     \n",
       "102430                                       NaN   \n",
       "413949                        Schoolwide Schools   \n",
       "433672                                       NaN   \n",
       "415831  \"Title  Part A Improving Basic Programs\"   \n",
       "\n",
       "                                Text_1  \n",
       "109283  STAFF DEV AND INSTR MEDIA       \n",
       "102430             INSTRUCTIONAL STAFF  \n",
       "413949                             NaN  \n",
       "433672             INSTRUCTIONAL STAFF  \n",
       "415831                  MISCELLANEOUS   \n",
       "\n",
       "[5 rows x 25 columns]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Function', 'Use', 'Sharing', 'Reporting', 'Student_Type',\n",
       "       'Position_Type', 'Object_Type', 'Pre_K', 'Operating_Status',\n",
       "       'Object_Description', 'Text_2', 'SubFund_Description',\n",
       "       'Job_Title_Description', 'Text_3', 'Text_4', 'Sub_Object_Description',\n",
       "       'Location_Description', 'FTE', 'Function_Description',\n",
       "       'Facility_or_Department', 'Position_Extra', 'Total',\n",
       "       'Program_Description', 'Fund_Description', 'Text_1'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>FTE</th>\n",
       "      <th>Total</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>126071.000000</td>\n",
       "      <td>3.957220e+05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.426794</td>\n",
       "      <td>1.310586e+04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.573576</td>\n",
       "      <td>3.682254e+05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>-0.087551</td>\n",
       "      <td>-8.746631e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.000792</td>\n",
       "      <td>7.379770e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.130927</td>\n",
       "      <td>4.612300e+02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.652662e+03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>46.800000</td>\n",
       "      <td>1.297000e+08</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 FTE         Total\n",
       "count  126071.000000  3.957220e+05\n",
       "mean        0.426794  1.310586e+04\n",
       "std         0.573576  3.682254e+05\n",
       "min        -0.087551 -8.746631e+07\n",
       "25%         0.000792  7.379770e+01\n",
       "50%         0.130927  4.612300e+02\n",
       "75%         1.000000  3.652662e+03\n",
       "max        46.800000  1.297000e+08"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "FTE: Stands for \"full-time equivalent\". If the budget item is associated to an employee, this number tells us the percentage of full-time that the employee works. A value of 1 means the associated employee works for the school full-time. A value close to 0 means the item is associated to a part-time or contracted employee.\n",
    "\n",
    "Total: Stands for the total cost of the expenditure. This number tells us how much the budget item cost.\n",
    "\n",
    "The variance in expenditure is high. Makes sense---some purchases are cheap, some are expensive. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZkAAAEKCAYAAADAVygjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAFd5JREFUeJzt3X+wJlV95/H3xxnlx0YBRbME0MFk\n4jqxEkAkuMbEiMEBFdTyB5YKWkQ2iqsmWVd0LXFVtnRXl4RIoiiUA6simqxMFJdFFI278mP8ERDQ\nYoIoI1ZIBEc3oyDjd//oc53reOfevsOc++OZ96vq1u0+ffp5vk/XzHymu89zOlWFJEk93G+xC5Ak\nTS5DRpLUjSEjSerGkJEkdWPISJK6MWQkSd0YMpKkbgwZSVI3howkqZuVi13AQtt///1r1apVi12G\nJC0bX/rSl/65qh66M/vudiGzatUqNmzYsNhlSNKykeRbO7uvl8skSd0YMpKkbgwZSVI3howkqRtD\nRpLUjSEjSerGkJEkdWPISJK6MWQkSd3sdiFz/Xc2s+r0Ty52GZK0W9jtQkaStHAMGUlSN4aMJKkb\nQ0aS1I0hI0nqxpCRJHVjyEiSujFkJEndGDKSpG4MGUlSN4aMJKkbQ0aS1I0hI0nqxpCRJHVjyEiS\nujFkJEndrOz9BklWABuA71TV05McAlwEPBj4MvDiqronyR7ABcBjge8Bz6+qW9trvB44BdgKvKqq\nLmvta4E/B1YA76+qt/f+PJK0XCyFBzQuxJnMq4Gbpq2/AzirqlYDdzGEB+33XVX1a8BZrR9J1gAn\nAr8BrAX+MsmKFl7nAMcCa4AXtL6SpCWia8gkOQh4GvD+th7gycDHWpd1wDPb8gltnbb96Nb/BOCi\nqrq7qr4JbASObD8bq+qWqrqH4ezohJ6fR5I0P73PZP4M+I/AT9v6Q4DvV9W9bX0TcGBbPhC4DaBt\n39z6/6x9u3121C5JWiK6hUySpwN3VNWXpjfP0LXm2Dbf9plqOTXJhiQbtm7ZPEvVkqRdqeeN/ycA\nxyc5DtgTeBDDmc2+SVa2s5WDgNtb/03AwcCmJCuBfYA7p7VPmb7Pjtp/TlWdC5wLsMcBq2cMIknS\nrtftTKaqXl9VB1XVKoYb95+pqhcCnwWe07qdDFzSlte3ddr2z1RVtfYTk+zRRqatBq4BrgVWJzkk\nyQPae6zv9XkkSfPXfQjzDF4HXJTkbcBXgPNa+3nAhUk2MpzBnAhQVTckuRi4EbgXOK2qtgIkeSVw\nGcMQ5vOr6oYF/SSSpFktSMhU1ZXAlW35FoaRYdv3+THw3B3sfyZw5gztlwKX7sJSJUm7kN/4lyR1\nY8hIkroxZCRJ3RgykqRuDBlJUjeGjCSpG0NGktSNISNJ6saQkSR1Y8hIkroxZCRJ3RgykqRuDBlJ\nUjeGjCSpG0NGktSNISNJ6saQkSR1Y8hIkroxZCRJ3RgykqRuDBlJUjeGjCSpG0NGktSNISNJ6saQ\nkSR1Y8hIkroxZCRJ3RgykqRuDBlJUjeGjCSpG0NGktSNISNJ6saQkSR1Y8hIkroxZCRJ3RgykqRu\nDBlJUjeGjCSpm24hk2TPJNck+fskNyT5z639kCRXJ7k5yUeSPKC179HWN7btq6a91utb+zeSPHVa\n+9rWtjHJ6b0+iyRp5/Q8k7kbeHJV/RZwKLA2yVHAO4Czqmo1cBdwSut/CnBXVf0acFbrR5I1wInA\nbwBrgb9MsiLJCuAc4FhgDfCC1leStER0C5ka/L+2ev/2U8CTgY+19nXAM9vyCW2dtv3oJGntF1XV\n3VX1TWAjcGT72VhVt1TVPcBFra8kaYnoek+mnXF8FbgDuBz4B+D7VXVv67IJOLAtHwjcBtC2bwYe\nMr19u3121D5THacm2ZBkw9Ytm3fFR5MkjdA1ZKpqa1UdChzEcObx6Jm6td/Zwbb5ts9Ux7lVdURV\nHbFi733mLlyStEssyOiyqvo+cCVwFLBvkpVt00HA7W15E3AwQNu+D3Dn9Pbt9tlRuyRpieg5uuyh\nSfZty3sBTwFuAj4LPKd1Oxm4pC2vb+u07Z+pqmrtJ7bRZ4cAq4FrgGuB1W202gMYBges7/V5JEnz\nt3LuLjvtAGBdGwV2P+DiqvpEkhuBi5K8DfgKcF7rfx5wYZKNDGcwJwJU1Q1JLgZuBO4FTquqrQBJ\nXglcBqwAzq+qGzp+HknSPHULmaq6DjhshvZbGO7PbN/+Y+C5O3itM4EzZ2i/FLj0PhcrSerCb/xL\nkroZFTJJHtO7EEnS5Bl7JvOeNkXMK6Zu5kuSNJdRIVNVvwO8kGHI8IYkH0ryB10rkyQte6PvyVTV\nzcAbgdcBvwecneTrSZ7dqzhJ0vI29p7MbyY5i+F7Lk8GnlFVj27LZ3WsT5K0jI0dwvxu4H3AG6rq\nR1ONVXV7kjd2qUyStOyNDZnjgB9N+xLk/YA9q2pLVV3YrTpJ0rI29p7Mp4G9pq3v3dokSdqhsSGz\n57Rnw9CW9+5TkiRpUowNmX9JcvjUSpLHAj+apb8kSaPvybwG+GiSqan0DwCe36ckSdKkGBUyVXVt\nkn8DPIrhYWFfr6qfdK1MkrTszWcW5scBq9o+hyWhqi7oUpUkaSKMCpkkFwK/CnwV2NqaCzBkJEk7\nNPZM5ghgTXtSpSRJo4wdXfY14F/3LESSNHnGnsnsD9yY5Brg7qnGqjq+S1WSpIkwNmTe3LMISdJk\nGjuE+XNJHgGsrqpPJ9kbWNG3NEnScjd2qv+XAR8D3tuaDgQ+3qsoSdJkGHvj/zTgCcAP4GcPMHtY\nr6IkSZNhbMjcXVX3TK0kWcnwPRlJknZobMh8LskbgL2S/AHwUeBv+5UlSZoEY0PmdOCfgOuBfwdc\nCvhETEnSrMaOLvspw+OX39e3HEnSJBk7d9k3meEeTFU9cpdXJEmaGPOZu2zKnsBzgQfv+nIkSZNk\n1D2ZqvretJ/vVNWfAU/uXJskaZkbe7ns8Gmr92M4s3lgl4okSRNj7OWyd01bvhe4FXjeLq9GkjRR\nxo4u+/3ehUiSJs/Yy2V/Mtv2qvrvu6YcSdIkmc/osscB69v6M4DPA7f1KEqSNBnm89Cyw6vqhwBJ\n3gx8tKr+sFdhkqTlb+y0Mg8H7pm2fg+wapdXI0maKGPPZC4ErknyPxm++f8s4IJuVUmSJsLYL2Oe\nCbwUuAv4PvDSqvovs+2T5OAkn01yU5Ibkry6tT84yeVJbm6/92vtSXJ2ko1Jrpv+3ZwkJ7f+Nyc5\neVr7Y5Nc3/Y5O0nmfwgkSb2MvVwGsDfwg6r6c2BTkkPm6H8v8KdV9WjgKOC0JGsYZnS+oqpWA1e0\ndYBjgdXt51Tgr2AIJeAM4LeBI4EzpoKp9Tl12n5r5/F5JEmdjX388hnA64DXt6b7A/9jtn2q6rtV\n9eW2/EPgJobHNp8ArGvd1gHPbMsnABfU4Cpg3yQHAE8FLq+qO6vqLuByYG3b9qCq+mJVFcPlu6nX\nkiQtAWPPZJ4FHA/8C0BV3c48ppVJsgo4DLga+OWq+m57ne+y7THOB/LzQ6I3tbbZ2jfN0C5JWiLG\nhsw97WyhAJL8q7FvkOSXgL8GXlNVP5it6wxttRPtM9VwapINSTZs3bJ5rpIlSbvI2JC5OMl7GS5h\nvQz4NCMeYJbk/gwB88Gq+pvW/I/tUhft9x2tfRNw8LTdDwJun6P9oBnaf0FVnVtVR1TVESv23meu\nsiVJu8jY0WXvBD7GEBiPAt5UVX8x2z5tpNd5wE3bTTuzHpgaIXYycMm09pPaKLOjgM3tctplwDFJ\n9ms3/I8BLmvbfpjkqPZeJ017LUnSEjDn92SSrGD4R/0pDDfdx3oC8GLg+iRfbW1vAN7OcGZ0CvBt\nhgegAVwKHAdsBLYwDJmmqu5M8lbg2tbvLVV1Z1t+OfABYC/gU+1HkrREzBkyVbU1yZYk+1TV6Bsa\nVfUFZr5vAnD0DP0LOG0Hr3U+cP4M7RuAx4ytSZK0sMZ+4//HDGckl9NGmAFU1au6VCVJmghjQ+aT\n7UeSpNFmDZkkD6+qb1fVutn6SZI0k7lGl318aiHJX3euRZI0YeYKmek37h/ZsxBJ0uSZK2RqB8uS\nJM1prhv/v5XkBwxnNHu1Zdp6VdWDulYnSVrWZg2ZqlqxUIVIkibPfJ4nI0nSvIz9nszEWXX64n3t\n59a3P23R3luSFpJnMpKkbgwZSVI3howkqRtDRpLUjSEjSerGkJEkdWPISJK6MWQkSd0YMpKkbgwZ\nSVI3howkqRtDRpLUjSEjSerGkJEkdWPISJK6MWQkSd0YMpKkbgwZSVI3howkqRtDRpLUjSEjSerG\nkJEkdWPISJK6MWQkSd0YMpKkbgwZSVI3howkqRtDRpLUTbeQSXJ+kjuSfG1a24OTXJ7k5vZ7v9ae\nJGcn2ZjkuiSHT9vn5Nb/5iQnT2t/bJLr2z5nJ0mvzyJJ2jk9z2Q+AKzdru104IqqWg1c0dYBjgVW\nt59Tgb+CIZSAM4DfBo4EzpgKptbn1Gn7bf9ekqRF1i1kqurzwJ3bNZ8ArGvL64BnTmu/oAZXAfsm\nOQB4KnB5Vd1ZVXcBlwNr27YHVdUXq6qAC6a9liRpiVjoezK/XFXfBWi/H9baDwRum9ZvU2ubrX3T\nDO0zSnJqkg1JNmzdsvk+fwhJ0jhL5cb/TPdTaifaZ1RV51bVEVV1xIq999nJEiVJ87XQIfOP7VIX\n7fcdrX0TcPC0fgcBt8/RftAM7ZKkJWShQ2Y9MDVC7GTgkmntJ7VRZkcBm9vltMuAY5Ls1274HwNc\n1rb9MMlRbVTZSdNeS5K0RKzs9cJJPgw8Cdg/ySaGUWJvBy5OcgrwbeC5rfulwHHARmAL8FKAqroz\nyVuBa1u/t1TV1GCClzOMYNsL+FT7kSQtId1CpqpesINNR8/Qt4DTdvA65wPnz9C+AXjMfalRktTX\nUrnxL0maQIaMJKkbQ0aS1I0hI0nqxpCRJHVjyEiSujFkJEndGDKSpG4MGUlSN4aMJKkbQ0aS1I0h\nI0nqxpCRJHVjyEiSujFkJEndGDKSpG4MGUlSN4aMJKkbQ0aS1I0hI0nqZuViFyDtLlad/slFe+9b\n3/60RXtv7d48k5EkdeOZzCLwf7SSdheeyUiSujFkJEndGDKSpG4MGUlSN4aMJKkbR5ftZhZzZBs4\nuk27l8X++7YUeCYjSerGMxntNvxfpbTwDBlJXS12uHuJdnEZMlpQi/0PjqSFZchImmj+x2ZxGTLS\nbsB/aLVYHF0mSerGkJEkdbPsQybJ2iTfSLIxyemLXY8kaZtlHTJJVgDnAMcCa4AXJFmzuFVJkqYs\n65ABjgQ2VtUtVXUPcBFwwiLXJElqlnvIHAjcNm19U2uTJC0By30Ic2Zoq1/olJwKnNpW7/7WO57+\nta5VLR/7A/+82EUsAR6HbTwW23gstnnUzu643ENmE3DwtPWDgNu371RV5wLnAiTZUFVHLEx5S5vH\nYuBx2MZjsY3HYpskG3Z23+V+uexaYHWSQ5I8ADgRWL/INUmSmmV9JlNV9yZ5JXAZsAI4v6puWOSy\nJEnNsg4ZgKq6FLh0Hruc26uWZchjMfA4bOOx2MZjsc1OH4tU/cJ9ckmSdonlfk9GkrSETWTIzDXV\nTJI9knykbb86yaqFr3JhjDgWf5LkxiTXJbkiySMWo86FMHYKoiTPSVJJJnZk0ZhjkeR57c/GDUk+\ntNA1LpQRf0cenuSzSb7S/p4ctxh1LoQk5ye5I8mMX/PI4Ox2rK5LcvicL1pVE/XDMADgH4BHAg8A\n/h5Ys12fVwDvacsnAh9Z7LoX8Vj8PrB3W3757nwsWr8HAp8HrgKOWOy6F/HPxWrgK8B+bf1hi133\nIh6Lc4GXt+U1wK2LXXfH4/G7wOHA13aw/TjgUwzfUTwKuHqu15zEM5kxU82cAKxryx8Djk4y0xc7\nl7s5j0VVfbaqtrTVqxi+azSJxk5B9FbgvwI/XsjiFtiYY/Ey4Jyqugugqu5Y4BoXyphjUcCD2vI+\nzPBdvElRVZ8H7pylywnABTW4Ctg3yQGzveYkhsyYqWZ+1qeq7gU2Aw9ZkOoW1nyn3TmF4X8pk2jO\nY5HkMODgqvrEQha2CMb8ufh14NeT/J8kVyVZu2DVLawxx+LNwIuSbGIYyfrvF6a0JWneU3kt+yHM\nMxgz1cyo6WgmwOjPmeRFwBHA73WtaPHMeiyS3A84C3jJQhW0iMb8uVjJcMnsSQxnt3+X5DFV9f3O\ntS20McfiBcAHqupdSR4PXNiOxU/7l7fkzPvfzkk8kxkz1czP+iRZyXAKPNsp4nI1atqdJE8B/hNw\nfFXdvUC1LbS5jsUDgccAVya5leF68/oJvfk/9u/IJVX1k6r6JvANhtCZNGOOxSnAxQBV9UVgT4Z5\nzXZHo/5NmW4SQ2bMVDPrgZPb8nOAz1S7qzVh5jwW7RLRexkCZlKvu8Mcx6KqNlfV/lW1qqpWMdyf\nOr6qdnrOpiVszN+RjzMMCiHJ/gyXz25Z0CoXxphj8W3gaIAkj2YImX9a0CqXjvXASW2U2VHA5qr6\n7mw7TNzlstrBVDNJ3gJsqKr1wHkMp7wbGc5gTly8ivsZeSz+G/BLwEfb2IdvV9Xxi1Z0JyOPxW5h\n5LG4DDgmyY3AVuC1VfW9xau6j5HH4k+B9yX5Y4ZLQy+Z0P+UkuTDDJdI92/3oM4A7g9QVe9huCd1\nHLAR2AK8dM7XnNBjJUlaAibxcpkkaYkwZCRJ3RgykqRuDBlJUjeGjCSpG0NGu70kD03yhSRfS/LM\nae2XJPmVnXitq9uMvU/cbtsT24zGX02y1yyvceXUl0CT3Nq+p7J9nycl+bfT1v8oyUnzqVVaCIaM\nNEwbsg54PPBagCTPAL5cVfOdDPFo4OtVdVhV/d12214IvLOqDq2qH93Hmp8E/Cxkquo9VXXBfXxN\naZczZCT4CbAXsAfw0zbV0GsYvqg6oySPaM/fmXoOz8OTHMowg/Nx25+tJPlD4HnAm5J8sJ2JfGLa\n9ncnecmYYjM8/+iPgD9u7/PEJG9O8h/a9iuTnJXk80luSvK4JH+T5OYkb5v2Oi9Kck17jfcmWTH2\ngEljGTISfAh4KvC/GGbcfQXDdOZbZtnn3a3PbwIfBM6uqq8Cb2J4Js/Pna1U1fsZpuR4bVW98L4U\nW1W3Au8Bzmrvs/0ZE8A9VfW7rd8lwGkMc7O9JMlD2vQozweeUFWHMnyr/z7VJc1k4qaVkearqjYD\nTwNIsh/wOuDZSd4H7Ae8q02MON3jgWe35QsZzmCWkqlpcq4HbpiaXyrJLQwTHP4O8Fjg2jad0F7A\nJM9dp0ViyEg/703AmQz3ab7EcJZzCW2yyFnMd36me/n5Kwl7ztY5yWkMDxKDYe6ouUzNpv3TactT\n6ysZpmxfV1WvH1WttJO8XCY1SVYDv1JVnwP2ZvgHuZg5AP4v2yZWfSHwhXm+3beANUn2SLIPbZbf\nHamqc9qlsUPbYIQfMjyeYGddATwnycMAkjw4ySPuw+tJMzJkpG3OBN7Ylj/M8ACzq4B3ztD3VcBL\nk1wHvBh49XzeqKpuY3hGyXUM93S+Ms9a/xZ41tSN/3nuS1XdyPBZ/3f7DJcDsz5GV9oZzsIsSerG\nMxlJUjeGjCSpG0NGktSNISNJ6saQkSR1Y8hIkroxZCRJ3RgykqRu/j9PrKN5d79zzAAAAABJRU5E\nrkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x13ae38e10>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df_train['FTE'].dropna().plot(kind = 'hist', xlim = [0, 1], bins = 500);\n",
    "plt.xlabel('% of full-time');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Bi-modal distribution! "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Making variables categorical.\n",
    "\n",
    "First let's look at object variables. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Function                   object\n",
       "Use                        object\n",
       "Sharing                    object\n",
       "Reporting                  object\n",
       "Student_Type               object\n",
       "Position_Type              object\n",
       "Object_Type                object\n",
       "Pre_K                      object\n",
       "Operating_Status           object\n",
       "Object_Description         object\n",
       "Text_2                     object\n",
       "SubFund_Description        object\n",
       "Job_Title_Description      object\n",
       "Text_3                     object\n",
       "Text_4                     object\n",
       "Sub_Object_Description     object\n",
       "Location_Description       object\n",
       "FTE                       float64\n",
       "Function_Description       object\n",
       "Facility_or_Department     object\n",
       "Position_Extra             object\n",
       "Total                     float64\n",
       "Program_Description        object\n",
       "Fund_Description           object\n",
       "Text_1                     object\n",
       "dtype: object"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "object     23\n",
       "float64     2\n",
       "dtype: int64"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.dtypes.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The object types are slow and don't work well with ML algorithms.\n",
    "\n",
    "For each of these rows, ERS attaches one label from each of 9 different categories as explained [here.](https://www.drivendata.org/competitions/46/box-plots-for-education-reboot/page/86/#labels_list)\n",
    "\n",
    "Next we convert these labels into categories type."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "labels = ['Function',\n",
    " 'Use',\n",
    " 'Sharing',\n",
    " 'Reporting',\n",
    " 'Student_Type',\n",
    " 'Position_Type',\n",
    " 'Object_Type',\n",
    " 'Pre_K',\n",
    " 'Operating_Status']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Function</th>\n",
       "      <th>Use</th>\n",
       "      <th>Sharing</th>\n",
       "      <th>Reporting</th>\n",
       "      <th>Student_Type</th>\n",
       "      <th>Position_Type</th>\n",
       "      <th>Object_Type</th>\n",
       "      <th>Pre_K</th>\n",
       "      <th>Operating_Status</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>134338</th>\n",
       "      <td>Teacher Compensation</td>\n",
       "      <td>Instruction</td>\n",
       "      <td>School Reported</td>\n",
       "      <td>School</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>Teacher</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>PreK-12 Operating</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>206341</th>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>Non-Operating</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>326408</th>\n",
       "      <td>Teacher Compensation</td>\n",
       "      <td>Instruction</td>\n",
       "      <td>School Reported</td>\n",
       "      <td>School</td>\n",
       "      <td>Unspecified</td>\n",
       "      <td>Teacher</td>\n",
       "      <td>Base Salary/Compensation</td>\n",
       "      <td>Non PreK</td>\n",
       "      <td>PreK-12 Operating</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>364634</th>\n",
       "      <td>Substitute Compensation</td>\n",
       "      <td>Instruction</td>\n",
       "      <td>School Reported</td>\n",
       "      <td>School</td>\n",
       "      <td>Unspecified</td>\n",
       "      <td>Substitute</td>\n",
       "      <td>Benefits</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>PreK-12 Operating</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47683</th>\n",
       "      <td>Substitute Compensation</td>\n",
       "      <td>Instruction</td>\n",
       "      <td>School Reported</td>\n",
       "      <td>School</td>\n",
       "      <td>Unspecified</td>\n",
       "      <td>Teacher</td>\n",
       "      <td>Substitute Compensation</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>PreK-12 Operating</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       Function          Use          Sharing Reporting  \\\n",
       "134338     Teacher Compensation  Instruction  School Reported    School   \n",
       "206341                 NO_LABEL     NO_LABEL         NO_LABEL  NO_LABEL   \n",
       "326408     Teacher Compensation  Instruction  School Reported    School   \n",
       "364634  Substitute Compensation  Instruction  School Reported    School   \n",
       "47683   Substitute Compensation  Instruction  School Reported    School   \n",
       "\n",
       "       Student_Type Position_Type               Object_Type     Pre_K  \\\n",
       "134338     NO_LABEL       Teacher                  NO_LABEL  NO_LABEL   \n",
       "206341     NO_LABEL      NO_LABEL                  NO_LABEL  NO_LABEL   \n",
       "326408  Unspecified       Teacher  Base Salary/Compensation  Non PreK   \n",
       "364634  Unspecified    Substitute                  Benefits  NO_LABEL   \n",
       "47683   Unspecified       Teacher   Substitute Compensation  NO_LABEL   \n",
       "\n",
       "         Operating_Status  \n",
       "134338  PreK-12 Operating  \n",
       "206341      Non-Operating  \n",
       "326408  PreK-12 Operating  \n",
       "364634  PreK-12 Operating  \n",
       "47683   PreK-12 Operating  "
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train[labels].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(400277, 9)"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train[labels].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'FTE',\n",
       " 'Facility_or_Department',\n",
       " 'Function_Description',\n",
       " 'Fund_Description',\n",
       " 'Job_Title_Description',\n",
       " 'Location_Description',\n",
       " 'Object_Description',\n",
       " 'Position_Extra',\n",
       " 'Program_Description',\n",
       " 'SubFund_Description',\n",
       " 'Sub_Object_Description',\n",
       " 'Text_1',\n",
       " 'Text_2',\n",
       " 'Text_3',\n",
       " 'Text_4',\n",
       " 'Total'}"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set(df_train.columns) - set(labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "categorize_label = lambda lab: lab.astype('category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train[labels] = df_train[labels].apply(categorize_label, axis = 'rows')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Function</th>\n",
       "      <th>Use</th>\n",
       "      <th>Sharing</th>\n",
       "      <th>Reporting</th>\n",
       "      <th>Student_Type</th>\n",
       "      <th>Position_Type</th>\n",
       "      <th>Object_Type</th>\n",
       "      <th>Pre_K</th>\n",
       "      <th>Operating_Status</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>134338</th>\n",
       "      <td>Teacher Compensation</td>\n",
       "      <td>Instruction</td>\n",
       "      <td>School Reported</td>\n",
       "      <td>School</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>Teacher</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>PreK-12 Operating</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>206341</th>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>Non-Operating</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>326408</th>\n",
       "      <td>Teacher Compensation</td>\n",
       "      <td>Instruction</td>\n",
       "      <td>School Reported</td>\n",
       "      <td>School</td>\n",
       "      <td>Unspecified</td>\n",
       "      <td>Teacher</td>\n",
       "      <td>Base Salary/Compensation</td>\n",
       "      <td>Non PreK</td>\n",
       "      <td>PreK-12 Operating</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>364634</th>\n",
       "      <td>Substitute Compensation</td>\n",
       "      <td>Instruction</td>\n",
       "      <td>School Reported</td>\n",
       "      <td>School</td>\n",
       "      <td>Unspecified</td>\n",
       "      <td>Substitute</td>\n",
       "      <td>Benefits</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>PreK-12 Operating</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47683</th>\n",
       "      <td>Substitute Compensation</td>\n",
       "      <td>Instruction</td>\n",
       "      <td>School Reported</td>\n",
       "      <td>School</td>\n",
       "      <td>Unspecified</td>\n",
       "      <td>Teacher</td>\n",
       "      <td>Substitute Compensation</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>PreK-12 Operating</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       Function          Use          Sharing Reporting  \\\n",
       "134338     Teacher Compensation  Instruction  School Reported    School   \n",
       "206341                 NO_LABEL     NO_LABEL         NO_LABEL  NO_LABEL   \n",
       "326408     Teacher Compensation  Instruction  School Reported    School   \n",
       "364634  Substitute Compensation  Instruction  School Reported    School   \n",
       "47683   Substitute Compensation  Instruction  School Reported    School   \n",
       "\n",
       "       Student_Type Position_Type               Object_Type     Pre_K  \\\n",
       "134338     NO_LABEL       Teacher                  NO_LABEL  NO_LABEL   \n",
       "206341     NO_LABEL      NO_LABEL                  NO_LABEL  NO_LABEL   \n",
       "326408  Unspecified       Teacher  Base Salary/Compensation  Non PreK   \n",
       "364634  Unspecified    Substitute                  Benefits  NO_LABEL   \n",
       "47683   Unspecified       Teacher   Substitute Compensation  NO_LABEL   \n",
       "\n",
       "         Operating_Status  \n",
       "134338  PreK-12 Operating  \n",
       "206341      Non-Operating  \n",
       "326408  PreK-12 Operating  \n",
       "364634  PreK-12 Operating  \n",
       "47683   PreK-12 Operating  "
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train[labels].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Function            category\n",
       "Use                 category\n",
       "Sharing             category\n",
       "Reporting           category\n",
       "Student_Type        category\n",
       "Position_Type       category\n",
       "Object_Type         category\n",
       "Pre_K               category\n",
       "Operating_Status    category\n",
       "dtype: object"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train[labels].dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 400277 entries, 134338 to 415831\n",
      "Data columns (total 9 columns):\n",
      "Function            400277 non-null category\n",
      "Use                 400277 non-null category\n",
      "Sharing             400277 non-null category\n",
      "Reporting           400277 non-null category\n",
      "Student_Type        400277 non-null category\n",
      "Position_Type       400277 non-null category\n",
      "Object_Type         400277 non-null category\n",
      "Pre_K               400277 non-null category\n",
      "Operating_Status    400277 non-null category\n",
      "dtypes: category(9)\n",
      "memory usage: 6.5 MB\n"
     ]
    }
   ],
   "source": [
    "df_train[labels].info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Counting unique labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Function            37\n",
       "Use                  8\n",
       "Sharing              5\n",
       "Reporting            3\n",
       "Student_Type         9\n",
       "Position_Type       25\n",
       "Object_Type         11\n",
       "Pre_K                3\n",
       "Operating_Status     3\n",
       "dtype: int64"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train[labels].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pandas.core.series.Series"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(df_train[labels].nunique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAFICAYAAABdiflbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3XmcXGWZ9vHfRUACCAIvUaMSg6zi\nFjAsiuOLERUXHBUV993ojAq+KiPiqIg6rrgxjhoHEBURRnBwcEFEXBBECYZNcBwljCgKKEsURRKu\n94/nVKg03enToes8la7r+/nUp+ucWs5Nk667nu1+ZJuIiBhdG9QOICIi6koiiIgYcUkEEREjLokg\nImLEJRFERIy4JIKIiBGXRBARMeKSCCIiRlwSQUTEiNuwdgBtbLPNNp4/f37tMCIi1itLly693vac\nyZ63XiSC+fPnc8EFF9QOIyJivSLpqjbPS9dQRMSISyKIiBhxSQQRESMuiSAiYsQlEUREjLgkgoiI\nEZdEEBEx4pIIIiJG3HqxoKyN+Yd9bdrea/n7njxt7xURMezSIoiIGHFJBBERIy6JICJixCURRESM\nuCSCiIgRl0QQETHikggiIkbcwBKBpNmSfizpIkmXSXpnc/6zkq6UtKy5LRhUDBERMblBLii7FVhk\n+0+SNgLOkfSN5rFDbX95gNeOiIiWBpYIbBv4U3O4UXPzoK4XERHrZqBjBJJmSVoGXAucafv85qH3\nSLpY0kckbTzBaxdLukDSBdddd90gw4yIGGkDTQS2V9leANwP2FPSg4G3ALsAewBbA2+e4LVLbC+0\nvXDOnDmDDDMiYqR1MmvI9o3Ad4H9bV/j4lbgOGDPLmKIiIjxDXLW0BxJWzb3NwH2A66QNLc5J+Bp\nwKWDiiEiIiY3yFlDc4HjJc2iJJyTbZ8u6TuS5gAClgGvHmAMERExiUHOGroY2G2c84sGdc2IiJi6\nrCyOiBhxSQQRESMuiSAiYsQlEUREjLgkgoiIEZdEEBEx4pIIIiJGXBJBRMSISyKIiBhxSQQRESMu\niSAiYsQlEUREjLgkgoiIEZdEEBEx4pIIIiJGXBJBRMSISyKIiBhxSQQRESMuiSAiYsQNLBFImi3p\nx5IuknSZpHc257eTdL6kX0g6SdLdBhVDRERMbpAtgluBRbYfBiwA9pe0N/B+4CO2dwRuAF4+wBgi\nImISkyYCSdtL2ri5v6+kgyVtOdnrXPypOdyouRlYBHy5OX888LR1ijwiIqZFmxbBKcAqSTsAxwDb\nAV9s8+aSZklaBlwLnAn8ErjR9srmKVcD953gtYslXSDpguuuu67N5SIiYh20SQS3Nx/cTwc+avv/\nAXPbvLntVbYXAPcD9gQeON7TJnjtEtsLbS+cM2dOm8tFRMQ6aJMIbpP0XODFwOnNuY2mchHbNwLf\nBfYGtpS0YfPQ/YDfTuW9IiJierVJBC8FHgG8x/aVkrYDvjDZiyTN6Y0lSNoE2A+4HDgbeGbztBcD\np61L4BERMT02nOwJtn8m6c3AvOb4SuB9Ld57LnC8pFmUhHOy7dMl/Qz4kqR3Az+ljDtEREQlkyYC\nSQcAHwLuBmwnaQFwpO2nru11ti8Gdhvn/K8o4wURETEE2nQNHUH54L4RwPYyysyhiIiYAdokgpW2\nbxpzbtyZPhERsf6ZtGsIuFTS84BZknYEDgbOHWxYERHRlTYtgtcBD6KUjDgRuBl4/SCDioiI7rSZ\nNXQL8NbmFhERM0ybWUNnM86YgO1FA4koIiI61WaM4E1992cDBwIrJ3huRESsZ9p0DS0dc+qHkr43\noHgiIqJjbbqGtu473AB4OHDvgUUUERGdatM1tJQyRiBKl9CVZDOZiIgZo03XUFYRR0TMYBMmAknP\nWNsLbZ86/eFERETX1tYiOGAtjxlIIoiImAEmTAS2X9plIBERUUebwWIkPZlSZmJ275ztIwcVVERE\ndGfSWkOSPgUcRKk5JOBZwP0HHFdERHSkTdG5R9p+EXCD7XdStq3cdrBhRUREV9okgr80P2+RdB/g\nNrIxTUTEjNFmjOD0ZhP6DwIXUmYMfWagUUVERGcmbRHYfpftG22fQhkb2MX22yd7naRtJZ0t6XJJ\nl0k6pDl/hKTfSFrW3J501/8zIiJiXbWpNXQRcBJwku1fUjaoaWMl8EbbF0raHFgq6czmsY/Y/tA6\nRRwREdOqzRjBUykf6idL+omkN0maN9mLbF9j+8Lm/grgcuC+dynaiIiYdm1qDV0FfAD4QLNn8duA\n9wOz2l5E0nxgN+B8YB/gtZJeBFxAaTXcMM5rFgOLAebNmzTvRIyE+Yd9bdrea/n7njxt7xXrtzYt\nAiTNl/RPwJeAXYB/ansBSXcHTgFeb/tm4JPA9sAC4BrgqPFeZ3uJ7YW2F86ZM6ft5SIiYorajBGc\nD2wEnAw8y/av2r65pI0oSeCEXpE627/ve/wzwOlTDToiIqZPm+mjL7Z9xVTfWJKAY4DLbX+47/xc\n29c0h08HLp3qe0dExPRpM0Yw5STQ2Ad4IXCJpGXNucOB50paQFmPsBx41Tq+f0RETINWRefWhe1z\nKLWJxvr6oK4ZERFT12qwOCIiZq421Uc3lfS2ZmAXSTtKesrgQ4uIiC60aREcR1lN/Ijm+Grg3QOL\nKCIiOtUmEWxv+wOUqqPY/gvj9/1HRMR6qE0i+JukTSizfJC0Pe3rDUVExJBrM2voHcA3gW0lnUCZ\nFvqSQQYVERHdabOO4ExJFwJ7U7qEDrF9/cAji4iITrQpMfHo5u6K5ueukrD9/cGFFRERXWnTNXRo\n3/3ZwJ7AUmDRQCKKiIhOtekaOqD/WNK2lLLUERExA6zLyuKrgQdPdyAREVFHmzGCo2mmjlISxwLg\nokEGFRER3WkzRnBB3/2VwIm2fzigeCIiomNtxgiO7yKQiIioo03X0CXc0TW0xkOAbT902qOKiIjO\ntOka+kbz8/PNz+cDtwBpKUREzABtEsE+tvfpOz5M0g9tHzmooCIiojttpo9uJulRvQNJjwQ2G1xI\nERHRpTYtgpcDx0q6R3N8I/CywYUUERFdajNraCnwMElbALJ9U5s3blYgfw64N3A7sMT2xyRtDZwE\nzKdsXv9s2zesW/gREXFXTZgIJL3A9hckvWHMeQBsf3iS914JvNH2hZI2B5ZKOpNSwvos2++TdBhw\nGPDmu/DfEBERd8HaWgS9cYDN1+WNbV8DXNPcXyHpcuC+wN8D+zZPOx74LkkEERHVTJgIbH+6+fnO\nu3oRSfOB3YDzgXs1SQLb10i65wSvWQwsBpg3b95dDSEiIibQZkHZHOCVlD791c+33WrAWNLdgVOA\n19u+ude1NBnbS4AlAAsXLhxvQVtEREyDNrOGTgN+AHwbWDWVN5e0ESUJnGD71Ob07yXNbVoDc4Fr\np/KeERExvdokgk1tT7kPX+Wr/zHA5WMGlr8KvBh4X/PztKm+d0RETJ82C8pOl/SkdXjvfYAXAosk\nLWtuT6IkgMdJ+gXwuOY4IiIqadMiOAQ4XNKtwG3cUWxui7W9yPY5zXPH89gpRRkREQPTZkHZOk0f\njYiI9UObWUOPHu+87e9PfzgREdG1Nl1Dh/bdnw3sCSwFFg0kooiI6FSbrqED+o+bGkIfGFhEERHR\nqTazhsa6GnjwdAcSERF1tBkjOJo7tqrcAFgAXDTIoCIiojttxggu6Lu/EjjR9g8HFE9ERHSszRhB\n9iaOiJjB1mWMICIiZpAkgoiIETdhIpD0+ebnId2FExERXVtbi+Dhku4PvEzSVpK27r91FWBERAzW\n2gaLPwV8E3gAZSVxfwE5N+cjImI9N2GLwPbHbT8QONb2A2xv13dLEoiImCHaTB/9B0kPA/6uOfV9\n2xcPNqyIiOjKpLOGJB0MnADcs7mdIOl1gw4sIiK60WZl8SuAvWz/GUDS+4HzgKMHGVhERHSjzToC\nseam9auYeOexiIhYz7RpERwHnC/pK83x0yib0kdExAwwaYvA9oeBlwJ/BG4AXmr7o5O9TtKxkq6V\ndGnfuSMk/WbMZvYREVFRmxYBti8ELpzie38W+Ffgc2POf8T2h6b4XhERMSADqzXU7Gn8x0G9f0RE\nTI8aRedeK+niputoq4meJGmxpAskXXDdddd1GV9ExEhZayKQNEvSt6fxep8EtqfscnYNcNRET7S9\nxPZC2wvnzJkzjSFERES/tSYC26uAWyTdYzouZvv3tlfZvh34DLDndLxvRESsuzaDxX8FLpF0JvDn\n3knbB0/1YpLm2r6mOXw6cOnanh8REYPXJhF8rblNiaQTgX2BbSRdDbwD2FfSAkr10uXAq6b6vhER\nMb1a7VksaRNgnu2ft31j288d53QWokVEDJk2RecOAJZR9iZA0gJJXx10YBER0Y0200ePoAzq3ghg\nexmw3QBjioiIDrVJBCtt3zTmnAcRTEREdK/NYPGlkp4HzJK0I3AwcO5gw4qIiK60aRG8DngQcCtw\nInAz8PpBBhUREd1pM2voFuCtzYY0tr1i8GFFRERX2swa2kPSJcDFlIVlF0l6+OBDi4iILrQZIzgG\n+EfbPwCQ9CjKZjUPHWRgERHRjTZjBCt6SQDA9jlAuociImaICVsEknZv7v5Y0qcpA8UGDgK+O/jQ\nIiKiC2vrGhpbIvodffezjiAiYoaYMBHYfkyXgURERB2TDhZL2hJ4ETC///nrUoY6IiKGT5tZQ18H\nfgRcAtw+2HAiIqJrbRLBbNtvGHgkERFRRZvpo5+X9EpJcyVt3bsNPLKIiOhEmxbB34APAm/ljtlC\nBh4wqKAiIqI7bRLBG4AdbF8/6GAiYv0z/7Ap72Q7oeXve/K0vVe016Zr6DLglkEHEhERdbRpEawC\nlkk6m1KKGph8+qikY4GnANfafnBzbmvgJMpU1OXAs23fsE6RR0TEtGjTIvhP4D2UzWiW9t0m81lg\n/zHnDgPOsr0jcFZzHBERFbXZj+D4dXlj29+XNH/M6b8H9m3uH0+pWfTmdXn/iIiYHm1WFl/JOLWF\nbK/LrKF72b6mef01ku65lusuBhYDzJs3bx0uFRERbbQZI1jYd3828Cxg4OsIbC8BlgAsXLgwRe4i\nIgZk0jEC23/ou/3G9keBRet4vd9LmgvQ/Lx2Hd8nIiKmSZuuod37DjegtBA2X8frfRV4MfC+5udp\n6/g+ERExTdp0DfXvS7CSZtrnZC+SdCJlYHgbSVdT9jN4H3CypJcD/0vpZoqIiIrazBpap30JbD93\ngoceuy7vFxERg9Gma2hj4EDuvB/BkYMLK6K+lE6IUdGma+g04CbKIrJbJ3luRESsZ9okgvvZHrtC\nOCIiZog2JSbOlfSQgUcSERFVtGkRPAp4SbPC+FZAgG0/dKCRRUREJ9okgicOPIoZKoONEbE+aDN9\n9KouAomIiDrajBFERMQMlkQQETHikggiIkZcEkFExIhLIoiIGHFJBBERIy6JICJixCURRESMuCSC\niIgRl0QQETHikggiIkZcEkFExIhrU3102klaDqwAVgErbS+sEUdERFRKBI3H2L6+4vUjIoJ0DUVE\njLxaLQID35Jk4NO2l4x9gqTFwGKAefPmdRzezDZdG+Zks5yImaFWi2Af27tTdj97jaRHj32C7SW2\nF9peOGfOnO4jjIgYEVUSge3fNj+vBb4C7FkjjoiIqJAIJG0mafPefeDxwKVdxxEREUWNMYJ7AV+R\n1Lv+F21/s0IcERFBhURg+1fAw7q+bkREjC/TRyMiRlwSQUTEiEsiiIgYcUkEEREjLokgImLE1Sw6\nF7Fayl7EdBvGf1PDGBOkRRARMfKSCCIiRlwSQUTEiEsiiIgYcUkEEREjLokgImLEJRFERIy4JIKI\niBGXRBARMeKSCCIiRlwSQUTEiEsiiIgYcUkEEREjrkoikLS/pJ9L+h9Jh9WIISIiis4TgaRZwCeA\nJwK7As+VtGvXcURERFGjRbAn8D+2f2X7b8CXgL+vEEdERACy3e0FpWcC+9t+RXP8QmAv268d87zF\nwOLmcGfg59MUwjbA9dP0XtMlMbWTmNobxrgSUzvTGdP9bc+Z7Ek1dijTOOfulI1sLwGWTPvFpQts\nL5zu970rElM7iam9YYwrMbVTI6YaXUNXA9v2Hd8P+G2FOCIigjqJ4CfAjpK2k3Q34DnAVyvEERER\nVOgasr1S0muBM4BZwLG2L+swhGnvbpoGiamdxNTeMMaVmNrpPKbOB4sjImK4ZGVxRMSISyKIiBhx\nSQQRESMuiSAiYsSNTCKQdF9Jj5T06N5tCGLaRNLOteOIGDRJGw9BDI9by2Pv7zKWtZG0gaQturzm\nSCSC5n/yD4F/Bg5tbm+qHNMBwDLgm83xAklV11NI+vg4t3dJqlYLStIKSTePuf1a0lckPaBSTDtI\nOkPSRc3xQyW9pUYswx6XpD0lXQL8ojl+mKSjK4XzCUlP7j/RfOh+FnhYnZBWx/FFSVtI2gz4GfBz\nSYd2df2RSATA04CdbT/J9gHN7amVYzqCUoDvRgDby4D5FeMBmA0soPzR/gJ4KLA18HJJH60U04cp\nifu+lFXobwI+QylWeGylmP4deCdwe3N8CfCCSrH0G8a4Pg48BfgDgO2LgMdUiuXxwFGSngEgaTZl\nMetGwAGVYurZ1fbNlM+qrwPzgBd2dfEatYZq+BXlf/attQPps9L2TdJ4pZeq2QFYZHslgKRPAt8C\nHkf5UKlhf9t79R0vkfQj20dKOrxSTJvZPrf3/862Jd1WKZZ+wxjXBravGvPvfFWNQGwvl7QfcIak\ne1I+aM+3/YYa8YyxkaSNKIngX23fJqmzRV6jkghuAZZJOou+ZGD74Hohcamk5wGzJO0IHAycWzEe\nKN+6NwNuao43A+5je5WkWkn0dknPBr7cHD+z77FaqyH/IGm73vUlPQ34XaVY+g1jXL+WtCfgZi+S\n1wH/XSMQSbs3d/8J+BxwJvCF3nnbF9aIq/FpYDlwEfB9SfcHbu7q4iOxsljSi8c7b/v4rmPpkbQp\n8FZKc1WUkhvvsv3XijG9nDKO8t0mpkcD/wKcCBxhu7M+y76YHgB8DHgE5QPuR8D/A34DPNz2ORVi\n2oFSBmBv4DrgGuA5tpd3Hcuwx9V88/44sB/l39SZwGttd176WdLZa3nYthd1FkwLkjbstc4Hfq1R\nSAQATYG7nZrDn9uu3WRerfmmtFnTR1g7lrmUsQsBP7adyrATkHQPyt/QjbVj6Tesca0vJD3O9pkd\nX/Pt4523fWQX1x+JwWJJ+1IGPz8B/Bvw37Wnj46ZJXAZHc8SWIsNKN8m/wjsMAS/pzmSDpe0RNKx\nvVvlmLaS9GHKt9szJB0laauaMQ1rXJLmNzO8ftfcTpE0v2ZMLdSYSvrnvtsqyla+87u6+Ei0CCQt\nBZ5n++fN8U7AibYfXjGmZbYXSHo+8HDgzcBS2w+tGNP7gYMoiak388Q1Z1hJOhf4AbCUvkFG26dU\njOkMShfVF5pTzwP2sf34WjHBcMYl6TxKd9UJfTG9yvYjasU0GUk/tb1b5Rg2Br5q+wldXG9UBos3\n6iUBANv/3YzQ11R1lsAEetNsh2l21aa231w7iDG2sf2OvuN3Nl82ahvGuDawfVzf8Wcl/UO1aNqp\n/XcIsCnQ2TqZkegaAi6QdIykfZvbZyjfMGv6FHAlZWZO57MEJtCbZjtMTpf0pNpBjPE9lb23AWjm\npX+jYjw9wxjXdyS9SdL9VFb3vwH4r6ZbtNPVs8NM0iWSLm5ul1H2aP94Z9cfka6hjYHXAI+iDIJ+\nH/i3Gt98mz+E1YeUbx/XAecAv+5qlsB4JJ1CWWE5NNNsJa2gJMtbgdtofme2q32ISLoBuEcTj4G7\ncceUW9veOnGtjunXa3nYtud1FkxLkk61/YyOr3n/vsOVwO+7/CwYiUQwTCS9Y5zTWwNPoEzR/FLH\nIa02jNNsh1Ezy2tCtqssmBrWuIZNM3X7jcA8269s1vHsbPv0ijF93vYLJzs3sOvP5EQg6WTbz1ap\ndXKn/9CaA7NjSdoa+Lbt3Sd98giQtIvtK/oWAa2h5uIfSb3yFmd6iP6AhjEuST+ixHSi7RW14wGQ\ndBKla/hFth8saRPgPNsLKsZ0Yf/fvqQNgYtt79rJ9Yfk38tASJpr+5oxza7VbF/VdUxrU2u2wjAm\nTElLbC+eYBFQ1cU/kvYHXgrsDpwEfNb2/9SKp2cY45K0SxPTsygr54+zfVblmC6wvbD/703SRbY7\nLzynUhTwcGATSgUEKN2ffwOW2O6maKDtGX8D3t/mXOUYFwHfqXTtuc3P+493q/x7md3mXKXYtgJe\nC/yaMu70QmDDxDVuTLOAp1NWhF8JvA3YslIs51I+eC9sjrenLJ6s+ft5b83rz+gWQc/YZldz7mLX\n+aY73rfurYHfUpqqV3QdE6zuXz7D9n41rj+RCf7f3elc15qFWs8DXgRcD3yRMhlhx5q/w2GMS9Ku\nlFbBAcB3KGsKHgUcVOP/o8q+BP8M7EopqrgP8BLb3+06ljFxbQXsSKkCDIDt73dx7Rm9jqCZr/yP\nwPaSLu57aHPqFXh7yphjA3+w/ecawawOohSWu0XSPWzfNPkrBkvSvSlF8DaRtBuluQywBWWOdTWS\nTgYeQvmQPdD21c1DJ0j6aeJaI6bzgb9QxgnebvsvzUM/lLRPhXgEXAE8g1KTScAhrlD7aExcrwAO\noZRaX9bEdh6lp2Dw15/JLQKVmitbAe8FDut7aIXtP9aJang1HyR7U0oUrE5MrjB9tJnB9BJgIfAT\n7kgENwPH2z61Qkx72/6RpMczXAOyQxeXpGfYPlXSTrarVBudiKSlrlhVYDxNT8EewI9cKg7sArzT\n9kGdXH8I/s0MnKS9gcvczFqQtDllI4jz60Y2XIZt+qikDYDn2j5h0id3YBi6pMYzjHENY0w9kj5B\nGUj/Se1YeiT9xPYekpYBe9m+VU0Zmi6uP6O7hvp8kjKToufP45wbebU+8Cdi+3ZJr+KOOjUR0+Ex\nwKslLad8FvQWKdacTn61pC2B/wTObBYHdlb5d1RaBHfKrLUGi4dZs7DmvZRBtP4Bqyp7AzcxvY3S\nx3wSa3ZXdd61J+lGyiyccblScb5hjEvSLcB4U1erf+gO+3RySf+XskL8G+6oXP6otAh+JelgSisA\nygDyryrGM6yOA94BfITyreml3NE3X8vLmp+v6TtnOizI1ec64KgK153MMMZ1JfX3AV6Dyh7Fr6Zs\nyXoJcIwrlnTp17+K2Pb3eufoaN/iUWkR9HZJWkT5EDkLeL3ta6sGNmR6g2iSLrH9kObcD2z/Xe3Y\nhsGw9nsPY1y1FkeuTbOi+DZKWfMnAlfZPqRuVMU4K4tnAZe4o5XFI9EiaD7wn1M7jvXAX5sB2l9I\nei1l8c89awakUqr7HyjbZkLZRvPTXTWZx1je5knqfoer5W2e1HFcP2zzJEkv7nBsate+LzjHAD/u\n6LoT6l9ZLKlXfXj1yuLO4hiRFsEc4JWUHX9WJz/bL5voNaNI0h7A5cCWwLso/ZQfsP2jijH9O6U0\ndu/D4oXAKtuvqBXTZIbxGzoMZ1xdxjTOt+6h+X1Ieq+7KicxjpFoEQCnUZqD36Zvl6tYU990uj9R\nxgeGwR5eswbMdyRdVC2admqPq0xkGOPqMqaHjfnW3fsWXq20eTNwfWMvCUh6DGWDqOXAJ2z/rYs4\nRiURDOMuV0NHZQvPQyk1hvpbTtUKvAGrJG1v+5cAkh7A8CfzYW1mD2NcncVke61luis5mVKD6SZJ\nC4D/oMzcW0DZX72Tlu+oJILTJT3J9tdrBzLk/oOyc9pnGJ4P20OBsyX9ivLN7f4MT2sl7rphbKV0\naRPbvfUCLwCOtX1UM1a3rKsgRiURHAIcLmlodrkaUittf3Lyp3XH9lnN+oadKf/frnDlPZUlbTw2\nhjHnlncfVSvLu76gpO1sX7mWc60GlWew/kS4CHgLrF5M2V0QozBYHGunsikOwMHAtcBXWHOrymp1\nmZq53/9IqVZpyljPp2z/tWJMQ1kRtYnjkdx5UsTnKsYz3u9q6Gr91CLpY8Bc4BrgqcBOtm+TNBf4\nL9sLu4hjJFoEkh493vmuSryuB5ZSPmR7X0HeNObxaiuLgc8BK4Cjm+PnAp+nbHTSqWGuiAqrFyBt\nT+lS6HXtmfI77DqWXYAHAfeQ1L//7xb0rVoPXg8cREkGj+qbFn1v4K1dBTESiYDSz9wzG9iT8uFX\ncxB0mBwE/Nr2NbC6+NyBlK6EI+qFBZS9ZPtnDZ1dcdbQEygVUe8HfLjv/ArKXPDaFlLmyg9DM39n\nSsn1LVlzhfEKylTuoPRPA3fap9z2GmXDJZ1n+xGDimMkEoHtNZa6S9oW+EClcIbRp4D9YHXr6b3A\n6ygzF5YAz6wXGj/tlVlu4tuLSv3KzcKn4yUdaPuUGjFM4lLKN8lragdi+zTgNEmPsH1e7XhmgIG2\nokYiEYzjauDBtYMYIrP6xgEOouyVegpwSlMWt6a9gBdJ+t/meB5weVO/vVbxstMlPY8798UfWSGW\nftsAP5P0Y9Yc46lSDK/xakmX274RVu/CdVQWc07ZQFt5I5EIJB3NHb/IDSjfdId9UVKXZknasCnA\n9Vhgcd9jtf+N7F/5+uM5DbiJ0r1YdQbTGEfUDmAcD+0lAQDbNzTjKzFEav+Rd+WCvvsrgRNtj/q0\ntX4nAt+TdD2l5PMPACTtQPnAq8b2VZJ6e+4eJ2kbYPOxUxI7dj/bQ5egbH9P0r0oO11B2ZC9dmHF\nDSRtZfsGWD1DbVQ+d6bTQOeSzujpo5Lm2f7fyZ8ZzS5uc4Fvudk/uVlpfHfbF1aM6x2UQdCdbe8k\n6T7Af9jufL/bvpiWAEfbvqRWDOOR9Gzgg5TCfAL+DjjU9pcrxvQiytz4L1Na5c8G3mP787ViWh9J\nerDtSwf2/jM8EayewyzpFNsH1o4ppqYZo9gNuLBX1rj2pkKSfkapaX8lpWuo+mYrTVwXAY/rtQKa\nYovfHjPrqkZcu1Jm6Ak4y/bPasYzjCSt4M7jADdRejPeaHug+6fM9CZaf3Oq5lz4WHd/s21JBpC0\nWe2AKLXsh9EGY7qC/kAZE6tta+DPTdfenPFWGwcfpmxN+UXK59ZzKDPAfg4cC+w7yIsPwz+SQfIE\n92P9cbKkTwNbSnolpYLsv9cQGIfdAAAH0UlEQVQMqNnScFtgUXP/Fobjb+mbks6Q9BJJLwG+BlSt\nr9V07b2ZpnQCpaT4F+pFNLT2t/1p2yts32x7CfAk2ycBWw364jO9RdArO9tfchZSa2i9YftDkh4H\n3ExZpPT2jjd9uZP+cQvK9p69D7dq4xYAtg+VdGAThyjTgL9SMyZKZc3dgAsBbP9W0uZ1QxpKtzdj\nPL3xnP61OwP/EjujE8GQlp2NKWo++M+EsoWfpOfbPqFiSEP74dZb/1E7jj7D2LU3jJ4PfIxSetrA\nj4AXSNoEeO2gLz6jE0GsvyRtQdmw/r7AVymJ4DWUciHLgJqJYKg+3CSdY/tR4ww4DkPLd2zX3sso\nZc6jTzMYfMAED58z6OvP6FlDsf6SdBpwA3AeZZHbVsDdgENsV13tLOlNwI7A4yjlOF4GfNH20Wt9\n4YhquvYeT0lMZ9Tu2htGtbfTTSKIoSTpEt+x0fgs4Hpgnu0VdSMrhvHDTdLnbb9wsnMxfCSdS1nI\nuZS+TaG6qmmVrqEYVr1yvNheJenKYUkCsOa4xRB5UP+BpA2BKnX/19Jd1fMH4IO2/63j0IZV1e10\n0yKIoSRpFfDn3iGwCWWaZs2Nxif6UAOgVl+8pLdQymD3fkdQfk9/o8wcestEr61F0v8BzrW9c+1Y\nhoGkd1N+H1Wm+yYRREyRpCOB31E2yBFlxsfmtquWNpf03iH90N+dO3aYO6dXa1/S3N4eGKOu+ZKx\nGWWleufb6SYRREyRpPNt7zXZuQ7j2cX2Fc0H7p1UrhX1dspucqc2p55GqRX17loxxZ0lEURMUTOw\n9wnKzlKmbJ/5GtuPrBTPEtuLJZ09zsO2XW0nPkmXA7u52WO6mRd/oe0H1oppmAxLEs9gccTUPY+y\n+OdjlETww+ZcFbYXNz8fUyuGtVhO2V3rr83xxsAvq0UzfN5A2f/jqHEeMx1tp5sWQcQMIelZwDdt\nr5D0z8DuwLs8Zv/bjmLpbQY1j7I/Qm+G1X6UcYLndB3TMJM0u9dqWtu5gV0/iSBiaiQdxzizh2pv\nv9grz91s5PNe4EPA4TXGLiS9uLm7CaUW0+2U+fF/gdX7P0ejv2T+2s4NSrqGIqbu9L77sym1h35b\nKZZ+vYVITwY+afs0SUdUiuWLwHsoq66volRn3ZZSpO/wSjENHUn3ppRR2aTZwrNXOn8LYNPO4kiL\nIOKukbQBZQOYaoOyTRynA7+hdL88nPLt+8c1NqaR9BHg7sAbegsBm/pRHwJusf36rmMaRk3L6SWU\narb9W+quAD5r+9TxXjftcSQRRNw1knYGvmZ7h8pxbArsD1xi+xeS5gIPsf2tCrH8AtjJYz5gmnIh\nV9jeseuYhpmkA7sqJzGedA1FTNE4K4x/R9l8pSrbt0j6JfAESU8AflAjCdwRzp2/ZTblQvLtcwzb\np0h6MqVMyOy+80d2cf1h2FUpYr1ie3PbW/Tddqr5ba5H0iGU8tz3bG5fkPS6SuH8rNm4fg2SXgBc\nUSGeoSbpU8BBwOso4wTPAu7f2fXTNRQxNZLOsv3Yyc51TdLFwCNs/7k53gw4z/ZDK8RyX8pq4r9Q\nKmqaMo10E+Dptn/TdUzDrG/GV+/n3YFTbT++i+unayiiJUmzKTM5tpG0FWvO8LhPtcDuIPpKGDf3\nNcFzB6r5oN9L0iJKd4eAb9g+q0Y864HeeoFbJN2HUp11u64unkQQ0d6rgNdTPvSX9p1fQSk5Udtx\nwPmSevsUPw04pmI82P4O8J2aMawn/kvSlsAHKVugmg53ckvXUERLkvYArgaeafvoZurfgZQyCkfY\n/mPN+GCNSp8Cvl9jVXFMTTP9eG/b5zbHGwOzbd/UWQxJBBHtSLoQ2M/2HyU9mlJ07nXAAuCBtp9Z\nKa7ZwKuBHYBLgGNsr6wRS6wbSefZfkSt62fWUER7s/q+9R9E2fTlFNtvo3wI13I8ZUHSJcATKYu2\nYv3yLUkHSqoyppMxgoj2ZknasPm2/VhK1ciemn9Lu/bt73wM8OOKscS6eQNlY5pVkv5CxxvTJBFE\ntHci8D1J11OmRf4AQNIOQGf9uePo3995ZaUvlXEX2N685vUzRhAxBZL2BuYC3+qbr78TcPdaO4EN\n4/7OMTVNl9Dzge1sv0vStsBc25207pIIIiIqk/RJSqnuRbYf2KxT+ZbtPbq4frqGIiLq28v27pJ+\nCmD7Bkl36+rimTUUEVHfbU1lVgNImkNpIXQiiSAior6PA18B7iXpPcA5wL90dfGMEUREDAFJu1Cm\nJQN8x/blXV07YwQREcNhU6DXPbRJlxdO11BERGWS3k5ZIb41sA1wnKR/7uz66RqKiKhL0uXAbrb/\n2hxvAlxo+4FdXD8tgoiI+pbTt0UlsDHwy64unhZBRERlkv6TsoPbmc2p/Sgzh64FsH3wIK+fweKI\niPrOAM6irB1YBZzd5cWTCCIiKpG0IWW9wMuAqyjd9dtSdps73PZta3n5tMkYQUREPR+kzBTazvbD\nbe8GPAC4R/NYJzJGEBFRiaRfADt5zAdxU27iCts7dhFHWgQREfV4bBJoTq6iqTvUhSSCiIh6fibp\nRWNPSnoBcEVXQaRrKCKiEkn3BU6l7Hi3lNIK2INSYuLptn/TSRxJBBERdUlaBDyIsqvcZbbP6vT6\nSQQREaMtYwQRESMuiSAiYsQlEUREjLgkgoiIEff/AZhAOgqTdMtBAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1241fe828>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df_train[labels].nunique().plot(kind = 'bar');\n",
    "plt.ylabel('number of unique values');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Model building**\n",
    "\n",
    "As a first pass build a simple model using just the numeric columns."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "numeric_data = df_train.select_dtypes(include = [float]).fillna(-1000)\n",
    "# filling NaNs with a large negative number"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "numeric_columns = list(df_train.select_dtypes(include = [float]).columns.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['FTE', 'Total']"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "numeric_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>FTE</th>\n",
       "      <th>Total</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>134338</th>\n",
       "      <td>1.0</td>\n",
       "      <td>50471.810</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>206341</th>\n",
       "      <td>-1000.0</td>\n",
       "      <td>3477.860</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>326408</th>\n",
       "      <td>1.0</td>\n",
       "      <td>62237.130</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>364634</th>\n",
       "      <td>-1000.0</td>\n",
       "      <td>22.300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47683</th>\n",
       "      <td>-1000.0</td>\n",
       "      <td>54.166</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           FTE      Total\n",
       "134338     1.0  50471.810\n",
       "206341 -1000.0   3477.860\n",
       "326408     1.0  62237.130\n",
       "364634 -1000.0     22.300\n",
       "47683  -1000.0     54.166"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "numeric_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'NUMERIC_COLUMNS' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-61-f86d8591e6cf>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mNUMERIC_COLUMNS\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'NUMERIC_COLUMNS' is not defined"
     ]
    }
   ],
   "source": [
    "NUMERIC_COLUMNS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>FTE</th>\n",
       "      <th>Total</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>134338</th>\n",
       "      <td>1.0</td>\n",
       "      <td>50471.810</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>206341</th>\n",
       "      <td>NaN</td>\n",
       "      <td>3477.860</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>326408</th>\n",
       "      <td>1.0</td>\n",
       "      <td>62237.130</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>364634</th>\n",
       "      <td>NaN</td>\n",
       "      <td>22.300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47683</th>\n",
       "      <td>NaN</td>\n",
       "      <td>54.166</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        FTE      Total\n",
       "134338  1.0  50471.810\n",
       "206341  NaN   3477.860\n",
       "326408  1.0  62237.130\n",
       "364634  NaN     22.300\n",
       "47683   NaN     54.166"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train[numeric_columns].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_label_dummies = pd.get_dummies(df_train[labels])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Function_Aides Compensation</th>\n",
       "      <th>Function_Career &amp; Academic Counseling</th>\n",
       "      <th>Function_Communications</th>\n",
       "      <th>Function_Curriculum Development</th>\n",
       "      <th>Function_Data Processing &amp; Information Services</th>\n",
       "      <th>Function_Development &amp; Fundraising</th>\n",
       "      <th>Function_Enrichment</th>\n",
       "      <th>Function_Extended Time &amp; Tutoring</th>\n",
       "      <th>Function_Facilities &amp; Maintenance</th>\n",
       "      <th>Function_Facilities Planning</th>\n",
       "      <th>...</th>\n",
       "      <th>Object_Type_Rent/Utilities</th>\n",
       "      <th>Object_Type_Substitute Compensation</th>\n",
       "      <th>Object_Type_Supplies/Materials</th>\n",
       "      <th>Object_Type_Travel &amp; Conferences</th>\n",
       "      <th>Pre_K_NO_LABEL</th>\n",
       "      <th>Pre_K_Non PreK</th>\n",
       "      <th>Pre_K_PreK</th>\n",
       "      <th>Operating_Status_Non-Operating</th>\n",
       "      <th>Operating_Status_Operating, Not PreK-12</th>\n",
       "      <th>Operating_Status_PreK-12 Operating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>134338</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>206341</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>326408</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>364634</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47683</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 104 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        Function_Aides Compensation  Function_Career & Academic Counseling  \\\n",
       "134338                            0                                      0   \n",
       "206341                            0                                      0   \n",
       "326408                            0                                      0   \n",
       "364634                            0                                      0   \n",
       "47683                             0                                      0   \n",
       "\n",
       "        Function_Communications  Function_Curriculum Development  \\\n",
       "134338                        0                                0   \n",
       "206341                        0                                0   \n",
       "326408                        0                                0   \n",
       "364634                        0                                0   \n",
       "47683                         0                                0   \n",
       "\n",
       "        Function_Data Processing & Information Services  \\\n",
       "134338                                                0   \n",
       "206341                                                0   \n",
       "326408                                                0   \n",
       "364634                                                0   \n",
       "47683                                                 0   \n",
       "\n",
       "        Function_Development & Fundraising  Function_Enrichment  \\\n",
       "134338                                   0                    0   \n",
       "206341                                   0                    0   \n",
       "326408                                   0                    0   \n",
       "364634                                   0                    0   \n",
       "47683                                    0                    0   \n",
       "\n",
       "        Function_Extended Time & Tutoring  Function_Facilities & Maintenance  \\\n",
       "134338                                  0                                  0   \n",
       "206341                                  0                                  0   \n",
       "326408                                  0                                  0   \n",
       "364634                                  0                                  0   \n",
       "47683                                   0                                  0   \n",
       "\n",
       "        Function_Facilities Planning                 ...                  \\\n",
       "134338                             0                 ...                   \n",
       "206341                             0                 ...                   \n",
       "326408                             0                 ...                   \n",
       "364634                             0                 ...                   \n",
       "47683                              0                 ...                   \n",
       "\n",
       "        Object_Type_Rent/Utilities  Object_Type_Substitute Compensation  \\\n",
       "134338                           0                                    0   \n",
       "206341                           0                                    0   \n",
       "326408                           0                                    0   \n",
       "364634                           0                                    0   \n",
       "47683                            0                                    1   \n",
       "\n",
       "        Object_Type_Supplies/Materials  Object_Type_Travel & Conferences  \\\n",
       "134338                               0                                 0   \n",
       "206341                               0                                 0   \n",
       "326408                               0                                 0   \n",
       "364634                               0                                 0   \n",
       "47683                                0                                 0   \n",
       "\n",
       "        Pre_K_NO_LABEL  Pre_K_Non PreK  Pre_K_PreK  \\\n",
       "134338               1               0           0   \n",
       "206341               1               0           0   \n",
       "326408               0               1           0   \n",
       "364634               1               0           0   \n",
       "47683                1               0           0   \n",
       "\n",
       "        Operating_Status_Non-Operating  \\\n",
       "134338                               0   \n",
       "206341                               1   \n",
       "326408                               0   \n",
       "364634                               0   \n",
       "47683                                0   \n",
       "\n",
       "        Operating_Status_Operating, Not PreK-12  \\\n",
       "134338                                        0   \n",
       "206341                                        0   \n",
       "326408                                        0   \n",
       "364634                                        0   \n",
       "47683                                         0   \n",
       "\n",
       "        Operating_Status_PreK-12 Operating  \n",
       "134338                                   1  \n",
       "206341                                   0  \n",
       "326408                                   1  \n",
       "364634                                   1  \n",
       "47683                                    1  \n",
       "\n",
       "[5 rows x 104 columns]"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_label_dummies.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(400277, 104)"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_label_dummies.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(400277, 9)"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train[labels].shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** Doing the test-train split **\n",
    "\n",
    "Some labels don't occur very often, but we want to make sure that they appear in both the training and the test sets. A way to do this is using StratifiedShuffleSplit. However scikit-learn's StratifiedShuffleSplit works only for one target variable. Here we have 104 of them! \n",
    "\n",
    "To accomplish the stratified shuffle split for multiple target variable, I use the function written by [Peter Bull.](https://github.com/drivendataorg/box-plots-sklearn/blob/master/src/data/multilabel.py)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def multilabel_sample(y, size=1000, min_count=5, seed=None):\n",
    "    \"\"\" Takes a matrix of binary labels `y` and returns\n",
    "        the indices for a sample of size `size` if\n",
    "        `size` > 1 or `size` * len(y) if size =< 1.\n",
    "        The sample is guaranteed to have > `min_count` of\n",
    "        each label.\n",
    "    \"\"\"\n",
    "    try:\n",
    "        if (np.unique(y).astype(int) != np.array([0, 1])).all():\n",
    "            raise ValueError()\n",
    "    except (TypeError, ValueError):\n",
    "        raise ValueError('multilabel_sample only works with binary indicator matrices')\n",
    "\n",
    "    if (y.sum(axis=0) < min_count).any():\n",
    "        raise ValueError('Some classes do not have enough examples. Change min_count if necessary.')\n",
    "\n",
    "    if size <= 1:\n",
    "        size = np.floor(y.shape[0] * size)\n",
    "\n",
    "    if y.shape[1] * min_count > size:\n",
    "        msg = \"Size less than number of columns * min_count, returning {} items instead of {}.\"\n",
    "        warn(msg.format(y.shape[1] * min_count, size))\n",
    "        size = y.shape[1] * min_count\n",
    "\n",
    "    rng = np.random.RandomState(seed if seed is not None else np.random.randint(1))\n",
    "\n",
    "    if isinstance(y, pd.DataFrame):\n",
    "        choices = y.index\n",
    "        y = y.values\n",
    "    else:\n",
    "        choices = np.arange(y.shape[0])\n",
    "\n",
    "    sample_idxs = np.array([], dtype=choices.dtype)\n",
    "\n",
    "    # first, guarantee > min_count of each label\n",
    "    for j in range(y.shape[1]):\n",
    "        label_choices = choices[y[:, j] == 1]\n",
    "        label_idxs_sampled = rng.choice(label_choices, size=min_count, replace=False)\n",
    "        sample_idxs = np.concatenate([label_idxs_sampled, sample_idxs])\n",
    "\n",
    "    sample_idxs = np.unique(sample_idxs)\n",
    "\n",
    "    # now that we have at least min_count of each, we can just random sample\n",
    "    sample_count = int(size - sample_idxs.shape[0])\n",
    "\n",
    "    # get sample_count indices from remaining choices\n",
    "    remaining_choices = np.setdiff1d(choices, sample_idxs)\n",
    "    remaining_sampled = rng.choice(remaining_choices,\n",
    "                                   size=sample_count,\n",
    "                                   replace=False)\n",
    "\n",
    "    return np.concatenate([sample_idxs, remaining_sampled])\n",
    "\n",
    "\n",
    "def multilabel_sample_dataframe(df, labels, size, min_count=5, seed=None):\n",
    "    \"\"\" Takes a dataframe `df` and returns a sample of size `size` where all\n",
    "        classes in the binary matrix `labels` are represented at\n",
    "        least `min_count` times.\n",
    "    \"\"\"\n",
    "    idxs = multilabel_sample(labels, size=size, min_count=min_count, seed=seed)\n",
    "    return df.loc[idxs]\n",
    "\n",
    "\n",
    "def multilabel_train_test_split(X, Y, size, min_count=5, seed=None):\n",
    "    \"\"\" Takes a features matrix `X` and a label matrix `Y` and\n",
    "        returns (X_train, X_test, Y_train, Y_test) where all\n",
    "        classes in Y are represented at least `min_count` times.\n",
    "    \"\"\"\n",
    "    index = Y.index if isinstance(Y, pd.DataFrame) else np.arange(Y.shape[0])\n",
    "\n",
    "    test_set_idxs = multilabel_sample(Y, size=size, min_count=min_count, seed=seed)\n",
    "    train_set_idxs = np.setdiff1d(index, test_set_idxs)\n",
    "\n",
    "    test_set_mask = index.isin(test_set_idxs)\n",
    "    train_set_mask = ~test_set_mask\n",
    "\n",
    "    return (X[train_set_mask], X[test_set_mask], Y[train_set_mask], Y[test_set_mask])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The function multilabel_train_test_split() makes sure that at least min_count examples appear in each split. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = multilabel_train_test_split(numeric_data,\n",
    "                                                               df_label_dummies,\n",
    "                                                               size=0.2, \n",
    "                                                               seed=123)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 320222 entries, 134338 to 415831\n",
      "Data columns (total 2 columns):\n",
      "FTE      320222 non-null float64\n",
      "Total    320222 non-null float64\n",
      "dtypes: float64(2)\n",
      "memory usage: 7.3 MB\n"
     ]
    }
   ],
   "source": [
    "X_train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 80055 entries, 206341 to 72072\n",
      "Data columns (total 2 columns):\n",
      "FTE      80055 non-null float64\n",
      "Total    80055 non-null float64\n",
      "dtypes: float64(2)\n",
      "memory usage: 1.8 MB\n"
     ]
    }
   ],
   "source": [
    "X_test.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 320222 entries, 134338 to 415831\n",
      "Columns: 104 entries, Function_Aides Compensation to Operating_Status_PreK-12 Operating\n",
      "dtypes: uint8(104)\n",
      "memory usage: 34.2 MB\n"
     ]
    }
   ],
   "source": [
    "y_train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Function_Aides Compensation</th>\n",
       "      <th>Function_Career &amp; Academic Counseling</th>\n",
       "      <th>Function_Communications</th>\n",
       "      <th>Function_Curriculum Development</th>\n",
       "      <th>Function_Data Processing &amp; Information Services</th>\n",
       "      <th>Function_Development &amp; Fundraising</th>\n",
       "      <th>Function_Enrichment</th>\n",
       "      <th>Function_Extended Time &amp; Tutoring</th>\n",
       "      <th>Function_Facilities &amp; Maintenance</th>\n",
       "      <th>Function_Facilities Planning</th>\n",
       "      <th>...</th>\n",
       "      <th>Object_Type_Rent/Utilities</th>\n",
       "      <th>Object_Type_Substitute Compensation</th>\n",
       "      <th>Object_Type_Supplies/Materials</th>\n",
       "      <th>Object_Type_Travel &amp; Conferences</th>\n",
       "      <th>Pre_K_NO_LABEL</th>\n",
       "      <th>Pre_K_Non PreK</th>\n",
       "      <th>Pre_K_PreK</th>\n",
       "      <th>Operating_Status_Non-Operating</th>\n",
       "      <th>Operating_Status_Operating, Not PreK-12</th>\n",
       "      <th>Operating_Status_PreK-12 Operating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>134338</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>326408</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>364634</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47683</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>229958</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 104 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        Function_Aides Compensation  Function_Career & Academic Counseling  \\\n",
       "134338                            0                                      0   \n",
       "326408                            0                                      0   \n",
       "364634                            0                                      0   \n",
       "47683                             0                                      0   \n",
       "229958                            0                                      0   \n",
       "\n",
       "        Function_Communications  Function_Curriculum Development  \\\n",
       "134338                        0                                0   \n",
       "326408                        0                                0   \n",
       "364634                        0                                0   \n",
       "47683                         0                                0   \n",
       "229958                        0                                0   \n",
       "\n",
       "        Function_Data Processing & Information Services  \\\n",
       "134338                                                0   \n",
       "326408                                                0   \n",
       "364634                                                0   \n",
       "47683                                                 0   \n",
       "229958                                                0   \n",
       "\n",
       "        Function_Development & Fundraising  Function_Enrichment  \\\n",
       "134338                                   0                    0   \n",
       "326408                                   0                    0   \n",
       "364634                                   0                    0   \n",
       "47683                                    0                    0   \n",
       "229958                                   0                    0   \n",
       "\n",
       "        Function_Extended Time & Tutoring  Function_Facilities & Maintenance  \\\n",
       "134338                                  0                                  0   \n",
       "326408                                  0                                  0   \n",
       "364634                                  0                                  0   \n",
       "47683                                   0                                  0   \n",
       "229958                                  0                                  1   \n",
       "\n",
       "        Function_Facilities Planning                 ...                  \\\n",
       "134338                             0                 ...                   \n",
       "326408                             0                 ...                   \n",
       "364634                             0                 ...                   \n",
       "47683                              0                 ...                   \n",
       "229958                             0                 ...                   \n",
       "\n",
       "        Object_Type_Rent/Utilities  Object_Type_Substitute Compensation  \\\n",
       "134338                           0                                    0   \n",
       "326408                           0                                    0   \n",
       "364634                           0                                    0   \n",
       "47683                            0                                    1   \n",
       "229958                           0                                    0   \n",
       "\n",
       "        Object_Type_Supplies/Materials  Object_Type_Travel & Conferences  \\\n",
       "134338                               0                                 0   \n",
       "326408                               0                                 0   \n",
       "364634                               0                                 0   \n",
       "47683                                0                                 0   \n",
       "229958                               0                                 0   \n",
       "\n",
       "        Pre_K_NO_LABEL  Pre_K_Non PreK  Pre_K_PreK  \\\n",
       "134338               1               0           0   \n",
       "326408               0               1           0   \n",
       "364634               1               0           0   \n",
       "47683                1               0           0   \n",
       "229958               1               0           0   \n",
       "\n",
       "        Operating_Status_Non-Operating  \\\n",
       "134338                               0   \n",
       "326408                               0   \n",
       "364634                               0   \n",
       "47683                                0   \n",
       "229958                               0   \n",
       "\n",
       "        Operating_Status_Operating, Not PreK-12  \\\n",
       "134338                                        0   \n",
       "326408                                        0   \n",
       "364634                                        0   \n",
       "47683                                         0   \n",
       "229958                                        0   \n",
       "\n",
       "        Operating_Status_PreK-12 Operating  \n",
       "134338                                   1  \n",
       "326408                                   1  \n",
       "364634                                   1  \n",
       "47683                                    1  \n",
       "229958                                   1  \n",
       "\n",
       "[5 rows x 104 columns]"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 80055 entries, 206341 to 72072\n",
      "Columns: 104 entries, Function_Aides Compensation to Operating_Status_PreK-12 Operating\n",
      "dtypes: uint8(104)\n",
      "memory usage: 8.6 MB\n"
     ]
    }
   ],
   "source": [
    "y_test.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Training Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OneVsRestClassifier(estimator=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False),\n",
       "          n_jobs=1)"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf = OneVsRestClassifier(LogisticRegression())\n",
    "clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "OneVsRestClassifier treats each columns of y independently. Essentially, it fits a separate classifier for each of the columns. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Pretty dismal score. But not very surprising given that we threw out most of our data. Let's go ahead and test the predictions on the holdout test data. Note that because we are developing 'human-in-the-loop' system, we wanna output the probabilities. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/sushantmore/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py:2698: DtypeWarning: Columns (5,11) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    }
   ],
   "source": [
    "holdout = pd.read_csv('TestData.csv', index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Object_Description</th>\n",
       "      <th>Program_Description</th>\n",
       "      <th>SubFund_Description</th>\n",
       "      <th>Job_Title_Description</th>\n",
       "      <th>Facility_or_Department</th>\n",
       "      <th>Sub_Object_Description</th>\n",
       "      <th>Location_Description</th>\n",
       "      <th>FTE</th>\n",
       "      <th>Function_Description</th>\n",
       "      <th>Position_Extra</th>\n",
       "      <th>Text_4</th>\n",
       "      <th>Total</th>\n",
       "      <th>Text_2</th>\n",
       "      <th>Text_3</th>\n",
       "      <th>Fund_Description</th>\n",
       "      <th>Text_1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>180042</th>\n",
       "      <td>Student Meals/Room/Other</td>\n",
       "      <td>Basic Educational Services</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Line Item that is paid with Campus' money</td>\n",
       "      <td>School</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Instruction</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3999.910000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>General Fund</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28872</th>\n",
       "      <td>Extra Duty/Signing Bonus Pay</td>\n",
       "      <td>Undistributed</td>\n",
       "      <td>NaN</td>\n",
       "      <td>CHEERLEADER DIR</td>\n",
       "      <td>NaN</td>\n",
       "      <td>General</td>\n",
       "      <td>School</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Cocurricular &amp; Extra Curricular Activities</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3447.320213</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>General Fund</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>186915</th>\n",
       "      <td>Professional Salaries</td>\n",
       "      <td>Bilingual Education</td>\n",
       "      <td>NaN</td>\n",
       "      <td>T-EL 1ST BIL</td>\n",
       "      <td>NaN</td>\n",
       "      <td>General</td>\n",
       "      <td>School</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Instruction</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>52738.780869</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>General Fund</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>412396</th>\n",
       "      <td>Professional Salaries</td>\n",
       "      <td>Bilingual Education</td>\n",
       "      <td>NaN</td>\n",
       "      <td>T-EL 2ND BIL</td>\n",
       "      <td>NaN</td>\n",
       "      <td>General</td>\n",
       "      <td>School</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Instruction</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>69729.263191</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>General Fund</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>427740</th>\n",
       "      <td>Salaries for Support Personnel</td>\n",
       "      <td>Undistributed</td>\n",
       "      <td>NaN</td>\n",
       "      <td>CLERK III- SCH</td>\n",
       "      <td>NaN</td>\n",
       "      <td>General</td>\n",
       "      <td>School</td>\n",
       "      <td>1.0</td>\n",
       "      <td>School Leadership</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>29492.834215</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>General Fund</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    Object_Description         Program_Description  \\\n",
       "180042        Student Meals/Room/Other  Basic Educational Services   \n",
       "28872     Extra Duty/Signing Bonus Pay               Undistributed   \n",
       "186915           Professional Salaries         Bilingual Education   \n",
       "412396           Professional Salaries         Bilingual Education   \n",
       "427740  Salaries for Support Personnel               Undistributed   \n",
       "\n",
       "       SubFund_Description Job_Title_Description Facility_or_Department  \\\n",
       "180042                 NaN                   NaN                    NaN   \n",
       "28872                  NaN       CHEERLEADER DIR                    NaN   \n",
       "186915                 NaN          T-EL 1ST BIL                    NaN   \n",
       "412396                 NaN          T-EL 2ND BIL                    NaN   \n",
       "427740                 NaN        CLERK III- SCH                    NaN   \n",
       "\n",
       "                           Sub_Object_Description Location_Description  FTE  \\\n",
       "180042  Line Item that is paid with Campus' money              School   NaN   \n",
       "28872                                     General              School   NaN   \n",
       "186915                                    General              School   1.0   \n",
       "412396                                    General              School   1.0   \n",
       "427740                                    General              School   1.0   \n",
       "\n",
       "                              Function_Description Position_Extra Text_4  \\\n",
       "180042                                 Instruction            NaN    NaN   \n",
       "28872   Cocurricular & Extra Curricular Activities            NaN    NaN   \n",
       "186915                                 Instruction            NaN    NaN   \n",
       "412396                                 Instruction            NaN    NaN   \n",
       "427740                           School Leadership            NaN    NaN   \n",
       "\n",
       "               Total Text_2 Text_3 Fund_Description Text_1  \n",
       "180042   3999.910000    NaN    NaN     General Fund    NaN  \n",
       "28872    3447.320213    NaN    NaN     General Fund    NaN  \n",
       "186915  52738.780869    NaN    NaN     General Fund    NaN  \n",
       "412396  69729.263191    NaN    NaN     General Fund    NaN  \n",
       "427740  29492.834215    NaN    NaN     General Fund    NaN  "
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "holdout.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/sushantmore/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n"
     ]
    }
   ],
   "source": [
    "prediction = clf.predict_proba(holdout[numeric_columns].fillna(-1000))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[3.58422797e-02, 6.46624380e-03, 8.29889897e-04, ...,\n",
       "        1.69612505e-01, 1.99296715e-02, 8.10542978e-01],\n",
       "       [3.58482728e-02, 6.46610323e-03, 8.29901972e-04, ...,\n",
       "        1.69607062e-01, 1.99300220e-02, 8.10552526e-01],\n",
       "       [1.20946821e-01, 9.06524483e-03, 1.58707950e-03, ...,\n",
       "        9.59309999e-02, 5.10388042e-02, 9.28360476e-01],\n",
       "       ...,\n",
       "       [1.22222570e-01, 9.05171601e-03, 1.58866973e-03, ...,\n",
       "        9.57003802e-02, 5.10986945e-02, 9.28644820e-01],\n",
       "       [1.22275131e-01, 9.04889685e-03, 1.58830780e-03, ...,\n",
       "        9.57066370e-02, 5.10808770e-02, 9.28635308e-01],\n",
       "       [1.22159718e-01, 9.05011410e-03, 1.58816455e-03, ...,\n",
       "        9.57273883e-02, 5.10754822e-02, 9.28609739e-01]])"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(50064, 104)"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prediction.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(50064, 16)"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "holdout.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(400277, 25)"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note 25 - 16 = 9. We have 9 taget variables (before applying the get_dummies)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's write predictions to a dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "prediction_df = pd.DataFrame(data = prediction, columns=pd.get_dummies(df_train[labels]).columns, \\\n",
    "                            index = holdout.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Function_Aides Compensation</th>\n",
       "      <th>Function_Career &amp; Academic Counseling</th>\n",
       "      <th>Function_Communications</th>\n",
       "      <th>Function_Curriculum Development</th>\n",
       "      <th>Function_Data Processing &amp; Information Services</th>\n",
       "      <th>Function_Development &amp; Fundraising</th>\n",
       "      <th>Function_Enrichment</th>\n",
       "      <th>Function_Extended Time &amp; Tutoring</th>\n",
       "      <th>Function_Facilities &amp; Maintenance</th>\n",
       "      <th>Function_Facilities Planning</th>\n",
       "      <th>...</th>\n",
       "      <th>Object_Type_Rent/Utilities</th>\n",
       "      <th>Object_Type_Substitute Compensation</th>\n",
       "      <th>Object_Type_Supplies/Materials</th>\n",
       "      <th>Object_Type_Travel &amp; Conferences</th>\n",
       "      <th>Pre_K_NO_LABEL</th>\n",
       "      <th>Pre_K_Non PreK</th>\n",
       "      <th>Pre_K_PreK</th>\n",
       "      <th>Operating_Status_Non-Operating</th>\n",
       "      <th>Operating_Status_Operating, Not PreK-12</th>\n",
       "      <th>Operating_Status_PreK-12 Operating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>180042</th>\n",
       "      <td>0.035842</td>\n",
       "      <td>0.006466</td>\n",
       "      <td>0.000830</td>\n",
       "      <td>0.023918</td>\n",
       "      <td>0.008916</td>\n",
       "      <td>0.000147</td>\n",
       "      <td>0.032077</td>\n",
       "      <td>0.024406</td>\n",
       "      <td>0.052099</td>\n",
       "      <td>0.000069</td>\n",
       "      <td>...</td>\n",
       "      <td>0.010729</td>\n",
       "      <td>0.036846</td>\n",
       "      <td>0.116126</td>\n",
       "      <td>0.017360</td>\n",
       "      <td>0.831241</td>\n",
       "      <td>0.141031</td>\n",
       "      <td>0.027749</td>\n",
       "      <td>0.169613</td>\n",
       "      <td>0.019930</td>\n",
       "      <td>0.810543</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28872</th>\n",
       "      <td>0.035848</td>\n",
       "      <td>0.006466</td>\n",
       "      <td>0.000830</td>\n",
       "      <td>0.023919</td>\n",
       "      <td>0.008916</td>\n",
       "      <td>0.000147</td>\n",
       "      <td>0.032078</td>\n",
       "      <td>0.024406</td>\n",
       "      <td>0.052102</td>\n",
       "      <td>0.000069</td>\n",
       "      <td>...</td>\n",
       "      <td>0.010728</td>\n",
       "      <td>0.036959</td>\n",
       "      <td>0.116164</td>\n",
       "      <td>0.017361</td>\n",
       "      <td>0.831233</td>\n",
       "      <td>0.141041</td>\n",
       "      <td>0.027751</td>\n",
       "      <td>0.169607</td>\n",
       "      <td>0.019930</td>\n",
       "      <td>0.810553</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>186915</th>\n",
       "      <td>0.120947</td>\n",
       "      <td>0.009065</td>\n",
       "      <td>0.001587</td>\n",
       "      <td>0.028599</td>\n",
       "      <td>0.016042</td>\n",
       "      <td>0.056023</td>\n",
       "      <td>0.043858</td>\n",
       "      <td>0.031715</td>\n",
       "      <td>0.113907</td>\n",
       "      <td>0.017065</td>\n",
       "      <td>...</td>\n",
       "      <td>0.005622</td>\n",
       "      <td>0.136221</td>\n",
       "      <td>0.135391</td>\n",
       "      <td>0.016041</td>\n",
       "      <td>0.501655</td>\n",
       "      <td>0.472173</td>\n",
       "      <td>0.098601</td>\n",
       "      <td>0.095931</td>\n",
       "      <td>0.051039</td>\n",
       "      <td>0.928360</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>412396</th>\n",
       "      <td>0.120381</td>\n",
       "      <td>0.009071</td>\n",
       "      <td>0.001586</td>\n",
       "      <td>0.028573</td>\n",
       "      <td>0.016044</td>\n",
       "      <td>0.055813</td>\n",
       "      <td>0.043808</td>\n",
       "      <td>0.031688</td>\n",
       "      <td>0.113723</td>\n",
       "      <td>0.017034</td>\n",
       "      <td>...</td>\n",
       "      <td>0.005630</td>\n",
       "      <td>0.125189</td>\n",
       "      <td>0.134056</td>\n",
       "      <td>0.016029</td>\n",
       "      <td>0.502143</td>\n",
       "      <td>0.471525</td>\n",
       "      <td>0.098399</td>\n",
       "      <td>0.096034</td>\n",
       "      <td>0.051012</td>\n",
       "      <td>0.928233</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>427740</th>\n",
       "      <td>0.121725</td>\n",
       "      <td>0.009057</td>\n",
       "      <td>0.001588</td>\n",
       "      <td>0.028634</td>\n",
       "      <td>0.016038</td>\n",
       "      <td>0.056310</td>\n",
       "      <td>0.043926</td>\n",
       "      <td>0.031752</td>\n",
       "      <td>0.114158</td>\n",
       "      <td>0.017107</td>\n",
       "      <td>...</td>\n",
       "      <td>0.005612</td>\n",
       "      <td>0.152629</td>\n",
       "      <td>0.137236</td>\n",
       "      <td>0.016058</td>\n",
       "      <td>0.500987</td>\n",
       "      <td>0.473061</td>\n",
       "      <td>0.098879</td>\n",
       "      <td>0.095790</td>\n",
       "      <td>0.051075</td>\n",
       "      <td>0.928534</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 104 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        Function_Aides Compensation  Function_Career & Academic Counseling  \\\n",
       "180042                     0.035842                               0.006466   \n",
       "28872                      0.035848                               0.006466   \n",
       "186915                     0.120947                               0.009065   \n",
       "412396                     0.120381                               0.009071   \n",
       "427740                     0.121725                               0.009057   \n",
       "\n",
       "        Function_Communications  Function_Curriculum Development  \\\n",
       "180042                 0.000830                         0.023918   \n",
       "28872                  0.000830                         0.023919   \n",
       "186915                 0.001587                         0.028599   \n",
       "412396                 0.001586                         0.028573   \n",
       "427740                 0.001588                         0.028634   \n",
       "\n",
       "        Function_Data Processing & Information Services  \\\n",
       "180042                                         0.008916   \n",
       "28872                                          0.008916   \n",
       "186915                                         0.016042   \n",
       "412396                                         0.016044   \n",
       "427740                                         0.016038   \n",
       "\n",
       "        Function_Development & Fundraising  Function_Enrichment  \\\n",
       "180042                            0.000147             0.032077   \n",
       "28872                             0.000147             0.032078   \n",
       "186915                            0.056023             0.043858   \n",
       "412396                            0.055813             0.043808   \n",
       "427740                            0.056310             0.043926   \n",
       "\n",
       "        Function_Extended Time & Tutoring  Function_Facilities & Maintenance  \\\n",
       "180042                           0.024406                           0.052099   \n",
       "28872                            0.024406                           0.052102   \n",
       "186915                           0.031715                           0.113907   \n",
       "412396                           0.031688                           0.113723   \n",
       "427740                           0.031752                           0.114158   \n",
       "\n",
       "        Function_Facilities Planning                 ...                  \\\n",
       "180042                      0.000069                 ...                   \n",
       "28872                       0.000069                 ...                   \n",
       "186915                      0.017065                 ...                   \n",
       "412396                      0.017034                 ...                   \n",
       "427740                      0.017107                 ...                   \n",
       "\n",
       "        Object_Type_Rent/Utilities  Object_Type_Substitute Compensation  \\\n",
       "180042                    0.010729                             0.036846   \n",
       "28872                     0.010728                             0.036959   \n",
       "186915                    0.005622                             0.136221   \n",
       "412396                    0.005630                             0.125189   \n",
       "427740                    0.005612                             0.152629   \n",
       "\n",
       "        Object_Type_Supplies/Materials  Object_Type_Travel & Conferences  \\\n",
       "180042                        0.116126                          0.017360   \n",
       "28872                         0.116164                          0.017361   \n",
       "186915                        0.135391                          0.016041   \n",
       "412396                        0.134056                          0.016029   \n",
       "427740                        0.137236                          0.016058   \n",
       "\n",
       "        Pre_K_NO_LABEL  Pre_K_Non PreK  Pre_K_PreK  \\\n",
       "180042        0.831241        0.141031    0.027749   \n",
       "28872         0.831233        0.141041    0.027751   \n",
       "186915        0.501655        0.472173    0.098601   \n",
       "412396        0.502143        0.471525    0.098399   \n",
       "427740        0.500987        0.473061    0.098879   \n",
       "\n",
       "        Operating_Status_Non-Operating  \\\n",
       "180042                        0.169613   \n",
       "28872                         0.169607   \n",
       "186915                        0.095931   \n",
       "412396                        0.096034   \n",
       "427740                        0.095790   \n",
       "\n",
       "        Operating_Status_Operating, Not PreK-12  \\\n",
       "180042                                 0.019930   \n",
       "28872                                  0.019930   \n",
       "186915                                 0.051039   \n",
       "412396                                 0.051012   \n",
       "427740                                 0.051075   \n",
       "\n",
       "        Operating_Status_PreK-12 Operating  \n",
       "180042                            0.810543  \n",
       "28872                             0.810553  \n",
       "186915                            0.928360  \n",
       "412396                            0.928233  \n",
       "427740                            0.928534  \n",
       "\n",
       "[5 rows x 104 columns]"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prediction_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Notice how we have probabilities for each column. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Function</th>\n",
       "      <th>Use</th>\n",
       "      <th>Sharing</th>\n",
       "      <th>Reporting</th>\n",
       "      <th>Student_Type</th>\n",
       "      <th>Position_Type</th>\n",
       "      <th>Object_Type</th>\n",
       "      <th>Pre_K</th>\n",
       "      <th>Operating_Status</th>\n",
       "      <th>Object_Description</th>\n",
       "      <th>...</th>\n",
       "      <th>Sub_Object_Description</th>\n",
       "      <th>Location_Description</th>\n",
       "      <th>FTE</th>\n",
       "      <th>Function_Description</th>\n",
       "      <th>Facility_or_Department</th>\n",
       "      <th>Position_Extra</th>\n",
       "      <th>Total</th>\n",
       "      <th>Program_Description</th>\n",
       "      <th>Fund_Description</th>\n",
       "      <th>Text_1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>134338</th>\n",
       "      <td>Teacher Compensation</td>\n",
       "      <td>Instruction</td>\n",
       "      <td>School Reported</td>\n",
       "      <td>School</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>Teacher</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>PreK-12 Operating</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>KINDERGARTEN</td>\n",
       "      <td>50471.810</td>\n",
       "      <td>KINDERGARTEN</td>\n",
       "      <td>General Fund</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>206341</th>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>Non-Operating</td>\n",
       "      <td>CONTRACTOR SERVICES</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>RGN  GOB</td>\n",
       "      <td>NaN</td>\n",
       "      <td>UNDESIGNATED</td>\n",
       "      <td>3477.860</td>\n",
       "      <td>BUILDING IMPROVEMENT SERVICES</td>\n",
       "      <td>NaN</td>\n",
       "      <td>BUILDING IMPROVEMENT SERVICES</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>326408</th>\n",
       "      <td>Teacher Compensation</td>\n",
       "      <td>Instruction</td>\n",
       "      <td>School Reported</td>\n",
       "      <td>School</td>\n",
       "      <td>Unspecified</td>\n",
       "      <td>Teacher</td>\n",
       "      <td>Base Salary/Compensation</td>\n",
       "      <td>Non PreK</td>\n",
       "      <td>PreK-12 Operating</td>\n",
       "      <td>Personal Services - Teachers</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>TEACHER</td>\n",
       "      <td>62237.130</td>\n",
       "      <td>Instruction - Regular</td>\n",
       "      <td>General Purpose School</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>364634</th>\n",
       "      <td>Substitute Compensation</td>\n",
       "      <td>Instruction</td>\n",
       "      <td>School Reported</td>\n",
       "      <td>School</td>\n",
       "      <td>Unspecified</td>\n",
       "      <td>Substitute</td>\n",
       "      <td>Benefits</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>PreK-12 Operating</td>\n",
       "      <td>EMPLOYEE BENEFITS</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>UNALLOC BUDGETS/SCHOOLS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>PROFESSIONAL-INSTRUCTIONAL</td>\n",
       "      <td>22.300</td>\n",
       "      <td>GENERAL MIDDLE/JUNIOR HIGH SCH</td>\n",
       "      <td>NaN</td>\n",
       "      <td>REGULAR INSTRUCTION</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47683</th>\n",
       "      <td>Substitute Compensation</td>\n",
       "      <td>Instruction</td>\n",
       "      <td>School Reported</td>\n",
       "      <td>School</td>\n",
       "      <td>Unspecified</td>\n",
       "      <td>Teacher</td>\n",
       "      <td>Substitute Compensation</td>\n",
       "      <td>NO_LABEL</td>\n",
       "      <td>PreK-12 Operating</td>\n",
       "      <td>TEACHER COVERAGE FOR TEACHER</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NON-PROJECT</td>\n",
       "      <td>NaN</td>\n",
       "      <td>PROFESSIONAL-INSTRUCTIONAL</td>\n",
       "      <td>54.166</td>\n",
       "      <td>GENERAL HIGH SCHOOL EDUCATION</td>\n",
       "      <td>NaN</td>\n",
       "      <td>REGULAR INSTRUCTION</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                       Function          Use          Sharing Reporting  \\\n",
       "134338     Teacher Compensation  Instruction  School Reported    School   \n",
       "206341                 NO_LABEL     NO_LABEL         NO_LABEL  NO_LABEL   \n",
       "326408     Teacher Compensation  Instruction  School Reported    School   \n",
       "364634  Substitute Compensation  Instruction  School Reported    School   \n",
       "47683   Substitute Compensation  Instruction  School Reported    School   \n",
       "\n",
       "       Student_Type Position_Type               Object_Type     Pre_K  \\\n",
       "134338     NO_LABEL       Teacher                  NO_LABEL  NO_LABEL   \n",
       "206341     NO_LABEL      NO_LABEL                  NO_LABEL  NO_LABEL   \n",
       "326408  Unspecified       Teacher  Base Salary/Compensation  Non PreK   \n",
       "364634  Unspecified    Substitute                  Benefits  NO_LABEL   \n",
       "47683   Unspecified       Teacher   Substitute Compensation  NO_LABEL   \n",
       "\n",
       "         Operating_Status            Object_Description  \\\n",
       "134338  PreK-12 Operating                           NaN   \n",
       "206341      Non-Operating           CONTRACTOR SERVICES   \n",
       "326408  PreK-12 Operating  Personal Services - Teachers   \n",
       "364634  PreK-12 Operating             EMPLOYEE BENEFITS   \n",
       "47683   PreK-12 Operating  TEACHER COVERAGE FOR TEACHER   \n",
       "\n",
       "                    ...               Sub_Object_Description  \\\n",
       "134338              ...                                  NaN   \n",
       "206341              ...                                  NaN   \n",
       "326408              ...                                  NaN   \n",
       "364634              ...                                  NaN   \n",
       "47683               ...                                  NaN   \n",
       "\n",
       "       Location_Description  FTE     Function_Description  \\\n",
       "134338                  NaN  1.0                      NaN   \n",
       "206341                  NaN  NaN                 RGN  GOB   \n",
       "326408                  NaN  1.0                      NaN   \n",
       "364634                  NaN  NaN  UNALLOC BUDGETS/SCHOOLS   \n",
       "47683                   NaN  NaN              NON-PROJECT   \n",
       "\n",
       "       Facility_or_Department              Position_Extra      Total  \\\n",
       "134338                    NaN               KINDERGARTEN   50471.810   \n",
       "206341                    NaN                UNDESIGNATED   3477.860   \n",
       "326408                    NaN                     TEACHER  62237.130   \n",
       "364634                    NaN  PROFESSIONAL-INSTRUCTIONAL     22.300   \n",
       "47683                     NaN  PROFESSIONAL-INSTRUCTIONAL     54.166   \n",
       "\n",
       "                   Program_Description        Fund_Description  \\\n",
       "134338                    KINDERGARTEN            General Fund   \n",
       "206341   BUILDING IMPROVEMENT SERVICES                     NaN   \n",
       "326408           Instruction - Regular  General Purpose School   \n",
       "364634  GENERAL MIDDLE/JUNIOR HIGH SCH                     NaN   \n",
       "47683    GENERAL HIGH SCHOOL EDUCATION                     NaN   \n",
       "\n",
       "                               Text_1  \n",
       "134338                            NaN  \n",
       "206341  BUILDING IMPROVEMENT SERVICES  \n",
       "326408                            NaN  \n",
       "364634            REGULAR INSTRUCTION  \n",
       "47683             REGULAR INSTRUCTION  \n",
       "\n",
       "[5 rows x 25 columns]"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "134338                    KINDERGARTEN \n",
       "206341                     UNDESIGNATED\n",
       "326408                          TEACHER\n",
       "364634       PROFESSIONAL-INSTRUCTIONAL\n",
       "47683        PROFESSIONAL-INSTRUCTIONAL\n",
       "229958                     UNDESIGNATED\n",
       "417668           SUPPLIES AND MATERIALS\n",
       "126378                     UNDESIGNATED\n",
       "275539       PROFESSIONAL-INSTRUCTIONAL\n",
       "85262        PROFESSIONAL-INSTRUCTIONAL\n",
       "304569                              NaN\n",
       "330504                              NaN\n",
       "84272                           TEACHER\n",
       "64760                               NaN\n",
       "21870                               NaN\n",
       "18698                DEGREED SUBSTITUTE\n",
       "169454                              NaN\n",
       "169914                              NaN\n",
       "189701                     UNDESIGNATED\n",
       "43727                               NaN\n",
       "5614                                NaN\n",
       "291539       PROFESSIONAL-INSTRUCTIONAL\n",
       "307038                              NaN\n",
       "27645                               NaN\n",
       "126388               SUBSTITUTE TEACHER\n",
       "14962                   TEACHER -  DAYS\n",
       "84040                               NaN\n",
       "61639        PROFESSIONAL-INSTRUCTIONAL\n",
       "266302       PROFESSIONAL-INSTRUCTIONAL\n",
       "225768                       BUS DRIVER\n",
       "                      ...              \n",
       "333748                              NaN\n",
       "292626           DIR EXEC DIR SECRETARY\n",
       "343458       PROFESSIONAL-INSTRUCTIONAL\n",
       "128261       PROFESSIONAL-INSTRUCTIONAL\n",
       "186318       PROFESSIONAL-INSTRUCTIONAL\n",
       "210652               SUBSTITUTE TEACHER\n",
       "340354                 PARAPROFESSIONAL\n",
       "435205       PROFESSIONAL-INSTRUCTIONAL\n",
       "174827                 PARAPROFESSIONAL\n",
       "277504                     UNDESIGNATED\n",
       "248062                              NaN\n",
       "70455                               NaN\n",
       "203453                              NaN\n",
       "58170                SUBSTITUTE TEACHER\n",
       "72072                    ASST PRINCIPAL\n",
       "68311        PROFESSIONAL-INSTRUCTIONAL\n",
       "225892       PROFESSIONAL-INSTRUCTIONAL\n",
       "446383                              NaN\n",
       "155111       RADIO & SUBSTITUTE DRIVERS\n",
       "278248                              NaN\n",
       "397424                 PARAPROFESSIONAL\n",
       "375092               SUBSTITUTE TEACHER\n",
       "220181         REG FOOD SERVICE WORKER \n",
       "307423                              NaN\n",
       "46691        PROFESSIONAL-INSTRUCTIONAL\n",
       "109283                              NaN\n",
       "102430       PROFESSIONAL-INSTRUCTIONAL\n",
       "413949                   PARENT/TITLE I\n",
       "433672    OFFICE/ADMINISTRATIVE SUPPORT\n",
       "415831             CERTIFIED SUBSTITUTE\n",
       "Name: Position_Extra, Length: 400277, dtype: object"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train['Position_Extra']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Getting started on NLP\n",
    "\n",
    "#### The first thing to do is create a bag of words\n",
    "\n",
    "Let's start by creating bag of words for just one feature 'Position_Extra'. We split the token on alpha numerics. Before doing that we fill the NaNs with empy strings."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_train['Position_Extra'].fillna('', inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "TOKEN_ALPHANUMERIC = '[A-Za-z0-9]+(?=\\\\s+)'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from nltk import word_tokenize          \n",
    "from nltk.stem import WordNetLemmatizer \n",
    "class LemmaTokenizer(object):\n",
    "    def __init__(self):\n",
    "         self.wnl = WordNetLemmatizer()\n",
    "    def __call__(self, doc):\n",
    "        return [self.wnl.lemmatize(t) for t in word_tokenize(doc)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Instantiate the CountVectorizer: vec_alphanumeric\n",
    "vec_alphanumeric = CountVectorizer(token_pattern = TOKEN_ALPHANUMERIC, stop_words='english', tokenizer=LemmaTokenizer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CountVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "        lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
       "        ngram_range=(1, 1), preprocessor=None, stop_words='english',\n",
       "        strip_accents=None, token_pattern='[A-Za-z0-9]+(?=\\\\s+)',\n",
       "        tokenizer=<__main__.LemmaTokenizer object at 0x1308d78d0>,\n",
       "        vocabulary=None)"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vec_alphanumeric.fit(df_train['Position_Extra'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['1st',\n",
       " '2nd',\n",
       " '3rd',\n",
       " '4th',\n",
       " '56',\n",
       " '5th',\n",
       " '9th',\n",
       " 'ab',\n",
       " 'accountability',\n",
       " 'adaptive',\n",
       " 'addit',\n",
       " 'additional',\n",
       " 'adm',\n",
       " 'admin',\n",
       " 'administrative',\n",
       " 'adult',\n",
       " 'aide',\n",
       " 'air',\n",
       " 'alarm',\n",
       " 'alt',\n",
       " 'area',\n",
       " 'arra',\n",
       " 'art',\n",
       " 'arts',\n",
       " 'assessment',\n",
       " 'assistant',\n",
       " 'assistive',\n",
       " 'asst',\n",
       " 'athletic',\n",
       " 'attendance',\n",
       " 'attn',\n",
       " 'audiology',\n",
       " 'auto',\n",
       " 'avg',\n",
       " 'aviation',\n",
       " 'ayp',\n",
       " 'b',\n",
       " 'behavior',\n",
       " 'behavioral',\n",
       " 'bilingual',\n",
       " 'biology',\n",
       " 'bldg',\n",
       " 'blind',\n",
       " 'board',\n",
       " 'broadcast',\n",
       " 'building',\n",
       " 'buildings',\n",
       " 'bus',\n",
       " 'business',\n",
       " 'cafe',\n",
       " 'camera',\n",
       " 'campbell',\n",
       " 'campus',\n",
       " 'card',\n",
       " 'career',\n",
       " 'case',\n",
       " 'ccoach',\n",
       " 'cen',\n",
       " 'center',\n",
       " 'central',\n",
       " 'cert',\n",
       " 'certified',\n",
       " 'change',\n",
       " 'chemistry',\n",
       " 'chief',\n",
       " 'child',\n",
       " 'chinese',\n",
       " 'classified',\n",
       " 'clerk',\n",
       " 'clinic',\n",
       " 'cntrl',\n",
       " 'coach',\n",
       " 'college',\n",
       " 'communication',\n",
       " 'computer',\n",
       " 'condition',\n",
       " 'consultant',\n",
       " 'consumer',\n",
       " 'cont',\n",
       " 'contractual',\n",
       " 'correctionist',\n",
       " 'cosmetology',\n",
       " 'counselor',\n",
       " 'criminal',\n",
       " 'crossing',\n",
       " 'culinary',\n",
       " 'curr',\n",
       " 'curriculum',\n",
       " 'cus',\n",
       " 'cust',\n",
       " 'custodial',\n",
       " 'custodian',\n",
       " 'dance',\n",
       " 'data',\n",
       " 'days',\n",
       " 'deaf',\n",
       " 'degreed',\n",
       " 'department',\n",
       " 'depty',\n",
       " 'deputy',\n",
       " 'dir',\n",
       " 'director',\n",
       " 'disabilities',\n",
       " 'disorder',\n",
       " 'dispatcher',\n",
       " 'distributed',\n",
       " 'domestic',\n",
       " 'drama',\n",
       " 'driver',\n",
       " 'e',\n",
       " 'earth',\n",
       " 'ec',\n",
       " 'economics',\n",
       " 'ed',\n",
       " 'educ',\n",
       " 'education',\n",
       " 'educational',\n",
       " 'elem',\n",
       " 'empac',\n",
       " 'employee',\n",
       " 'eng',\n",
       " 'engineer',\n",
       " 'english',\n",
       " 'equip',\n",
       " 'es',\n",
       " 'esl',\n",
       " 'exec',\n",
       " 'executive',\n",
       " 'exploration',\n",
       " 'ext',\n",
       " 'facilitator',\n",
       " 'field',\n",
       " 'financial',\n",
       " 'fix',\n",
       " 'fixed',\n",
       " 'food',\n",
       " 'foods',\n",
       " 'foreman',\n",
       " 'french',\n",
       " 'fs',\n",
       " 'fsd',\n",
       " 'gen',\n",
       " 'general',\n",
       " 'generalist',\n",
       " 'geography',\n",
       " 'german',\n",
       " 'gifted',\n",
       " 'gnets',\n",
       " 'grade',\n",
       " 'graduation',\n",
       " 'graphic',\n",
       " 'grease',\n",
       " 'gross',\n",
       " 'guard',\n",
       " 'guidance',\n",
       " 'handicapped',\n",
       " 'head',\n",
       " 'health',\n",
       " 'hearing',\n",
       " 'heavy',\n",
       " 'high',\n",
       " 'history',\n",
       " 'home',\n",
       " 'homebound',\n",
       " 'horticulture',\n",
       " 'hospital',\n",
       " 'hourly',\n",
       " 'hr',\n",
       " 'hrly',\n",
       " 'hs',\n",
       " 'impaired',\n",
       " 'information',\n",
       " 'instruc',\n",
       " 'instruct',\n",
       " 'instructional',\n",
       " 'instrumental',\n",
       " 'inter',\n",
       " 'intermediate',\n",
       " 'intern',\n",
       " 'interpreter',\n",
       " 'interrelated',\n",
       " 'intervention',\n",
       " 'intr',\n",
       " 'intructional',\n",
       " 'iss',\n",
       " 'janitorial',\n",
       " 'japanese',\n",
       " 'job',\n",
       " 'jr',\n",
       " 'justice',\n",
       " 'k',\n",
       " 'kdg',\n",
       " 'kindergarten',\n",
       " 'ladder',\n",
       " 'lang',\n",
       " 'large',\n",
       " 'latin',\n",
       " 'lead',\n",
       " 'leadman',\n",
       " 'learning',\n",
       " 'librarian',\n",
       " 'licensed',\n",
       " 'lieutenant',\n",
       " 'life',\n",
       " 'literacy',\n",
       " 'mailroom',\n",
       " 'maintenance',\n",
       " 'man',\n",
       " 'manager',\n",
       " 'marketing',\n",
       " 'math',\n",
       " 'mechanic',\n",
       " 'mechanics',\n",
       " 'med',\n",
       " 'media',\n",
       " 'mental',\n",
       " 'mentally',\n",
       " 'mgr',\n",
       " 'mid',\n",
       " 'middle',\n",
       " 'mildly',\n",
       " 'minor',\n",
       " 'misc',\n",
       " 'mo',\n",
       " 'moderate',\n",
       " 'months',\n",
       " 'ms',\n",
       " 'music',\n",
       " 'nc',\n",
       " 'needs',\n",
       " 'network',\n",
       " 'night',\n",
       " 'non',\n",
       " 'nurse',\n",
       " 'occupancy',\n",
       " 'occupational',\n",
       " 'occupations',\n",
       " 'office',\n",
       " 'officer',\n",
       " 'oil',\n",
       " 'operating',\n",
       " 'operation',\n",
       " 'optional',\n",
       " 'orthopedically',\n",
       " 'overtime',\n",
       " 'pac',\n",
       " 'painter',\n",
       " 'para',\n",
       " 'paraprof',\n",
       " 'parent',\n",
       " 'partially',\n",
       " 'patrol',\n",
       " 'pay',\n",
       " 'payroll',\n",
       " 'pe',\n",
       " 'phys',\n",
       " 'physical',\n",
       " 'physics',\n",
       " 'police',\n",
       " 'political',\n",
       " 'primary',\n",
       " 'prin',\n",
       " 'principal',\n",
       " 'print',\n",
       " 'proc',\n",
       " 'production',\n",
       " 'prof',\n",
       " 'professional',\n",
       " 'program',\n",
       " 'programmer',\n",
       " 'property',\n",
       " 'psych',\n",
       " 'psychological',\n",
       " 'psychologist',\n",
       " 'public',\n",
       " 'pump',\n",
       " 'purchased',\n",
       " 'radio',\n",
       " 'reading',\n",
       " 'rec',\n",
       " 'receives',\n",
       " 'referral',\n",
       " 'reg',\n",
       " 'regular',\n",
       " 'repair',\n",
       " 'reqd',\n",
       " 'resource',\n",
       " 'retarded',\n",
       " 'retiree',\n",
       " 'rotc',\n",
       " 'router',\n",
       " 'sac',\n",
       " 'sal',\n",
       " 'sales',\n",
       " 'sasi',\n",
       " 'sat',\n",
       " 'sch',\n",
       " 'school',\n",
       " 'science',\n",
       " 'sec',\n",
       " 'secondary',\n",
       " 'secretary',\n",
       " 'security',\n",
       " 'self',\n",
       " 'senior',\n",
       " 'service',\n",
       " 'services',\n",
       " 'severely',\n",
       " 'shop',\n",
       " 'sighted',\n",
       " 'site',\n",
       " 'size',\n",
       " 'small',\n",
       " 'social',\n",
       " 'solutions',\n",
       " 'sp',\n",
       " 'spanish',\n",
       " 'spec',\n",
       " 'special',\n",
       " 'specialist',\n",
       " 'speech',\n",
       " 'sr',\n",
       " 'staff',\n",
       " 'start',\n",
       " 'structured',\n",
       " 'sub',\n",
       " 'subsidies',\n",
       " 'substitute',\n",
       " 'summer',\n",
       " 'super',\n",
       " 'supervisor',\n",
       " 'supplies',\n",
       " 'supply',\n",
       " 'support',\n",
       " 'supt',\n",
       " 'systems',\n",
       " 'talented',\n",
       " 'tap',\n",
       " 'teacher',\n",
       " 'teachers',\n",
       " 'team',\n",
       " 'tech',\n",
       " 'technical',\n",
       " 'technician',\n",
       " 'technicians',\n",
       " 'technology',\n",
       " 'telecommunication',\n",
       " 'temp',\n",
       " 'temporary',\n",
       " 'textbook',\n",
       " 'therapist',\n",
       " 'therapy',\n",
       " 'time',\n",
       " 'title',\n",
       " 'train',\n",
       " 'transition',\n",
       " 'transportation',\n",
       " 'travel',\n",
       " 'traveling',\n",
       " 'trip',\n",
       " 'truck',\n",
       " 'tv',\n",
       " 'used',\n",
       " 'visual',\n",
       " 'vocational',\n",
       " 'voe',\n",
       " 'warehouse',\n",
       " 'web',\n",
       " 'wkr',\n",
       " 'worker']"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vec_alphanumeric.get_feature_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['1st',\n",
       " '2nd',\n",
       " '3rd',\n",
       " '4th',\n",
       " '56',\n",
       " '5th',\n",
       " '9th',\n",
       " 'a',\n",
       " 'ab',\n",
       " 'accountability',\n",
       " 'adaptive',\n",
       " 'addit',\n",
       " 'additional',\n",
       " 'adm',\n",
       " 'admin',\n",
       " 'administrative',\n",
       " 'adult',\n",
       " 'aide',\n",
       " 'air',\n",
       " 'alarm',\n",
       " 'alt',\n",
       " 'and',\n",
       " 'any',\n",
       " 'area',\n",
       " 'arra',\n",
       " 'art',\n",
       " 'arts',\n",
       " 'assessment',\n",
       " 'assistant',\n",
       " 'assistive',\n",
       " 'asst',\n",
       " 'at',\n",
       " 'athletic',\n",
       " 'attendance',\n",
       " 'attn',\n",
       " 'audiology',\n",
       " 'auto',\n",
       " 'avg',\n",
       " 'aviation',\n",
       " 'ayp',\n",
       " 'b',\n",
       " 'behavior',\n",
       " 'behavioral',\n",
       " 'bilingual',\n",
       " 'biology',\n",
       " 'bldg',\n",
       " 'blind',\n",
       " 'board',\n",
       " 'broadcast',\n",
       " 'building',\n",
       " 'buildings',\n",
       " 'bus',\n",
       " 'business',\n",
       " 'cafe',\n",
       " 'camera',\n",
       " 'campbell',\n",
       " 'campus',\n",
       " 'card',\n",
       " 'career',\n",
       " 'case',\n",
       " 'ccoach',\n",
       " 'cen',\n",
       " 'center',\n",
       " 'central',\n",
       " 'cert',\n",
       " 'certified',\n",
       " 'change',\n",
       " 'chemistry',\n",
       " 'chief',\n",
       " 'child',\n",
       " 'chinese',\n",
       " 'classified',\n",
       " 'clerk',\n",
       " 'clinic',\n",
       " 'cntrl',\n",
       " 'co',\n",
       " 'coach',\n",
       " 'college',\n",
       " 'communication',\n",
       " 'computer',\n",
       " 'condition',\n",
       " 'consultant',\n",
       " 'consumer',\n",
       " 'cont',\n",
       " 'contractual',\n",
       " 'correctionist',\n",
       " 'cosmetology',\n",
       " 'counselor',\n",
       " 'criminal',\n",
       " 'crossing',\n",
       " 'culinary',\n",
       " 'curr',\n",
       " 'curriculum',\n",
       " 'cus',\n",
       " 'cust',\n",
       " 'custodial',\n",
       " 'custodian',\n",
       " 'dance',\n",
       " 'data',\n",
       " 'days',\n",
       " 'deaf',\n",
       " 'degreed',\n",
       " 'department',\n",
       " 'depty',\n",
       " 'deputy',\n",
       " 'dir',\n",
       " 'director',\n",
       " 'disabilities',\n",
       " 'disorder',\n",
       " 'dispatcher',\n",
       " 'distributed',\n",
       " 'domestic',\n",
       " 'drama',\n",
       " 'driver',\n",
       " 'e',\n",
       " 'earth',\n",
       " 'ec',\n",
       " 'economics',\n",
       " 'ed',\n",
       " 'educ',\n",
       " 'education',\n",
       " 'educational',\n",
       " 'elem',\n",
       " 'empac',\n",
       " 'employee',\n",
       " 'eng',\n",
       " 'engineer',\n",
       " 'english',\n",
       " 'equip',\n",
       " 'es',\n",
       " 'esl',\n",
       " 'exec',\n",
       " 'executive',\n",
       " 'exploration',\n",
       " 'ext',\n",
       " 'facilitator',\n",
       " 'field',\n",
       " 'financial',\n",
       " 'fix',\n",
       " 'fixed',\n",
       " 'food',\n",
       " 'foods',\n",
       " 'for',\n",
       " 'foreman',\n",
       " 'french',\n",
       " 'fs',\n",
       " 'fsd',\n",
       " 'full',\n",
       " 'gen',\n",
       " 'general',\n",
       " 'generalist',\n",
       " 'geography',\n",
       " 'german',\n",
       " 'gifted',\n",
       " 'gnets',\n",
       " 'grade',\n",
       " 'graduation',\n",
       " 'graphic',\n",
       " 'grease',\n",
       " 'gross',\n",
       " 'guard',\n",
       " 'guidance',\n",
       " 'handicapped',\n",
       " 'head',\n",
       " 'health',\n",
       " 'hearing',\n",
       " 'heavy',\n",
       " 'high',\n",
       " 'history',\n",
       " 'home',\n",
       " 'homebound',\n",
       " 'horticulture',\n",
       " 'hospital',\n",
       " 'hourly',\n",
       " 'hr',\n",
       " 'hrly',\n",
       " 'hs',\n",
       " 'i',\n",
       " 'impaired',\n",
       " 'information',\n",
       " 'instruc',\n",
       " 'instruct',\n",
       " 'instructional',\n",
       " 'instrumental',\n",
       " 'inter',\n",
       " 'intermediate',\n",
       " 'intern',\n",
       " 'interpreter',\n",
       " 'interrelated',\n",
       " 'intervention',\n",
       " 'intr',\n",
       " 'intructional',\n",
       " 'is',\n",
       " 'iss',\n",
       " 'janitorial',\n",
       " 'japanese',\n",
       " 'job',\n",
       " 'jr',\n",
       " 'justice',\n",
       " 'k',\n",
       " 'kdg',\n",
       " 'kindergarten',\n",
       " 'ladder',\n",
       " 'lang',\n",
       " 'large',\n",
       " 'latin',\n",
       " 'lead',\n",
       " 'leadman',\n",
       " 'learning',\n",
       " 'librarian',\n",
       " 'licensed',\n",
       " 'lieutenant',\n",
       " 'life',\n",
       " 'literacy',\n",
       " 'mailroom',\n",
       " 'maintenance',\n",
       " 'man',\n",
       " 'manager',\n",
       " 'marketing',\n",
       " 'math',\n",
       " 'mechanic',\n",
       " 'mechanics',\n",
       " 'med',\n",
       " 'media',\n",
       " 'mental',\n",
       " 'mentally',\n",
       " 'mgr',\n",
       " 'mid',\n",
       " 'middle',\n",
       " 'mildly',\n",
       " 'minor',\n",
       " 'misc',\n",
       " 'mo',\n",
       " 'moderate',\n",
       " 'months',\n",
       " 'ms',\n",
       " 'music',\n",
       " 'nc',\n",
       " 'needs',\n",
       " 'network',\n",
       " 'night',\n",
       " 'no',\n",
       " 'non',\n",
       " 'none',\n",
       " 'not',\n",
       " 'nurse',\n",
       " 'occupancy',\n",
       " 'occupational',\n",
       " 'occupations',\n",
       " 'of',\n",
       " 'office',\n",
       " 'officer',\n",
       " 'oil',\n",
       " 'operating',\n",
       " 'operation',\n",
       " 'optional',\n",
       " 'orthopedically',\n",
       " 'other',\n",
       " 'overtime',\n",
       " 'pac',\n",
       " 'painter',\n",
       " 'para',\n",
       " 'paraprof',\n",
       " 'parent',\n",
       " 'partially',\n",
       " 'patrol',\n",
       " 'pay',\n",
       " 'payroll',\n",
       " 'pe',\n",
       " 'phys',\n",
       " 'physical',\n",
       " 'physics',\n",
       " 'police',\n",
       " 'political',\n",
       " 'primary',\n",
       " 'prin',\n",
       " 'principal',\n",
       " 'print',\n",
       " 'proc',\n",
       " 'production',\n",
       " 'prof',\n",
       " 'professional',\n",
       " 'program',\n",
       " 'programmer',\n",
       " 'property',\n",
       " 'psych',\n",
       " 'psychological',\n",
       " 'psychologist',\n",
       " 'public',\n",
       " 'pump',\n",
       " 'purchased',\n",
       " 'radio',\n",
       " 'reading',\n",
       " 'rec',\n",
       " 'receives',\n",
       " 'referral',\n",
       " 'reg',\n",
       " 'regular',\n",
       " 'repair',\n",
       " 'reqd',\n",
       " 'resource',\n",
       " 'retarded',\n",
       " 'retiree',\n",
       " 'rotc',\n",
       " 'router',\n",
       " 'sac',\n",
       " 'sal',\n",
       " 'sales',\n",
       " 'sasi',\n",
       " 'sat',\n",
       " 'sch',\n",
       " 'school',\n",
       " 'science',\n",
       " 'sec',\n",
       " 'secondary',\n",
       " 'secretary',\n",
       " 'security',\n",
       " 'self',\n",
       " 'senior',\n",
       " 'service',\n",
       " 'services',\n",
       " 'severely',\n",
       " 'shop',\n",
       " 'sighted',\n",
       " 'site',\n",
       " 'size',\n",
       " 'small',\n",
       " 'social',\n",
       " 'solutions',\n",
       " 'sp',\n",
       " 'spanish',\n",
       " 'spec',\n",
       " 'special',\n",
       " 'specialist',\n",
       " 'speech',\n",
       " 'sr',\n",
       " 'staff',\n",
       " 'start',\n",
       " 'structured',\n",
       " 'sub',\n",
       " 'subsidies',\n",
       " 'substitute',\n",
       " 'summer',\n",
       " 'super',\n",
       " 'supervisor',\n",
       " 'supplies',\n",
       " 'supply',\n",
       " 'support',\n",
       " 'supt',\n",
       " 'systems',\n",
       " 'talented',\n",
       " 'tap',\n",
       " 'teacher',\n",
       " 'teachers',\n",
       " 'team',\n",
       " 'tech',\n",
       " 'technical',\n",
       " 'technician',\n",
       " 'technicians',\n",
       " 'technology',\n",
       " 'telecommunication',\n",
       " 'temp',\n",
       " 'temporary',\n",
       " 'textbook',\n",
       " 'therapist',\n",
       " 'therapy',\n",
       " 'time',\n",
       " 'title',\n",
       " 'train',\n",
       " 'transition',\n",
       " 'transportation',\n",
       " 'travel',\n",
       " 'traveling',\n",
       " 'trip',\n",
       " 'truck',\n",
       " 'tv',\n",
       " 'used',\n",
       " 'visual',\n",
       " 'vocational',\n",
       " 'voe',\n",
       " 'warehouse',\n",
       " 'web',\n",
       " 'who',\n",
       " 'wkr',\n",
       " 'worker']"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vec_alphanumeric.get_feature_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['&',\n",
       " '(',\n",
       " ')',\n",
       " ',',\n",
       " '-',\n",
       " '-2nd',\n",
       " '.',\n",
       " '1st',\n",
       " '2nd',\n",
       " '3rd',\n",
       " '4th',\n",
       " '56',\n",
       " '5th',\n",
       " '9th',\n",
       " 'ab',\n",
       " 'accountability',\n",
       " 'accountant',\n",
       " 'adaptive',\n",
       " 'addit',\n",
       " 'additional',\n",
       " 'addl',\n",
       " 'adm',\n",
       " 'admin',\n",
       " 'administrative',\n",
       " 'administrator',\n",
       " 'adult',\n",
       " 'aide',\n",
       " 'air',\n",
       " 'alarm',\n",
       " 'allotment',\n",
       " 'alt',\n",
       " 'analyst',\n",
       " 'anlayst',\n",
       " 'area',\n",
       " 'arra',\n",
       " 'art',\n",
       " 'assessment',\n",
       " 'asset',\n",
       " 'assistant',\n",
       " 'assistive',\n",
       " 'asst',\n",
       " 'asst-sp',\n",
       " 'athletic',\n",
       " 'attendance',\n",
       " 'attn',\n",
       " 'audiologist',\n",
       " 'audiology',\n",
       " 'auto',\n",
       " 'avg',\n",
       " 'aviation',\n",
       " 'ayp',\n",
       " 'b',\n",
       " 'bachelor',\n",
       " 'behavior',\n",
       " 'behavioral',\n",
       " 'bilingual',\n",
       " 'biology',\n",
       " 'birth/kind',\n",
       " 'bldg',\n",
       " 'blind',\n",
       " 'bnc-supervisor',\n",
       " 'board',\n",
       " 'bookkeeper',\n",
       " 'broadcast',\n",
       " 'building',\n",
       " 'building/property',\n",
       " 'bus',\n",
       " 'business',\n",
       " 'cafe',\n",
       " 'camera',\n",
       " 'campbell',\n",
       " 'campus',\n",
       " 'card',\n",
       " 'career',\n",
       " 'case',\n",
       " 'ccoach',\n",
       " 'cen',\n",
       " 'center',\n",
       " 'central',\n",
       " 'cert',\n",
       " 'certified',\n",
       " 'certifiedaddl',\n",
       " 'change',\n",
       " 'charge',\n",
       " 'chef',\n",
       " 'chemistry',\n",
       " 'chief',\n",
       " 'child',\n",
       " 'chinese',\n",
       " 'classified',\n",
       " 'clerical',\n",
       " 'clerical/bookkeeper',\n",
       " 'clerk',\n",
       " 'clinic',\n",
       " 'co-dispatcher',\n",
       " 'co-tchr',\n",
       " 'coach',\n",
       " 'college',\n",
       " 'comm',\n",
       " 'communication',\n",
       " 'computer',\n",
       " 'condition',\n",
       " 'const.sec.',\n",
       " 'consultant',\n",
       " 'cont',\n",
       " 'containd',\n",
       " 'contained',\n",
       " 'contract',\n",
       " 'contractual',\n",
       " 'corporal',\n",
       " 'correctionist',\n",
       " 'cosmetology',\n",
       " 'cost',\n",
       " 'counselor',\n",
       " 'craft',\n",
       " 'criminal',\n",
       " 'crossing',\n",
       " 'cti',\n",
       " 'cu',\n",
       " 'culinary',\n",
       " 'curr',\n",
       " 'curriculum',\n",
       " 'cust',\n",
       " 'custodial',\n",
       " 'custodian',\n",
       " 'dance',\n",
       " 'data',\n",
       " 'day',\n",
       " 'deaf',\n",
       " 'degreed',\n",
       " 'department',\n",
       " 'dept',\n",
       " 'depty',\n",
       " 'deputy',\n",
       " 'design',\n",
       " 'development',\n",
       " 'diagnostician',\n",
       " 'dietician',\n",
       " 'dir',\n",
       " 'director',\n",
       " 'directr',\n",
       " 'disability',\n",
       " 'disorder',\n",
       " 'dispatcher',\n",
       " 'distributed',\n",
       " 'doctoral',\n",
       " 'domestic',\n",
       " 'drama',\n",
       " 'driver',\n",
       " 'e',\n",
       " 'earth',\n",
       " 'ec',\n",
       " 'economics',\n",
       " 'ed',\n",
       " 'ed-arra',\n",
       " 'ed-gnets',\n",
       " 'ed-speech',\n",
       " 'educ',\n",
       " 'education',\n",
       " 'educational',\n",
       " 'eip-intermediate',\n",
       " 'eip-kindergarten',\n",
       " 'eip-primary',\n",
       " 'elem',\n",
       " 'elem/inter',\n",
       " 'elem/interm',\n",
       " 'elem/intermediate',\n",
       " 'elementary',\n",
       " 'empac',\n",
       " 'employee',\n",
       " 'energy',\n",
       " 'eng',\n",
       " 'engineer',\n",
       " 'engineering',\n",
       " 'english',\n",
       " 'equip',\n",
       " 'equp',\n",
       " 'esl',\n",
       " 'esol',\n",
       " 'examiner',\n",
       " 'exec',\n",
       " 'executive',\n",
       " 'exploration',\n",
       " 'f',\n",
       " 'facilitator',\n",
       " 'family/consumer',\n",
       " 'field',\n",
       " 'financial',\n",
       " 'fix',\n",
       " 'fixed',\n",
       " 'flat',\n",
       " 'food',\n",
       " 'foreman',\n",
       " 'french',\n",
       " 'fsd',\n",
       " 'gen',\n",
       " 'general',\n",
       " 'generalist',\n",
       " 'geography',\n",
       " 'german',\n",
       " 'gifted',\n",
       " 'grade',\n",
       " 'graduation',\n",
       " 'graphic',\n",
       " 'grease',\n",
       " 'gross',\n",
       " 'guard',\n",
       " 'guidance',\n",
       " 'h',\n",
       " 'handicapped',\n",
       " 'head',\n",
       " 'health',\n",
       " 'hearing',\n",
       " 'heavy',\n",
       " 'high',\n",
       " 'history',\n",
       " 'home',\n",
       " 'homebound',\n",
       " 'homeless',\n",
       " 'horticulture',\n",
       " 'hospital',\n",
       " 'hourly',\n",
       " 'hr',\n",
       " 'hrly',\n",
       " 'i/ext',\n",
       " 'ii/ext',\n",
       " 'iii/ext',\n",
       " 'impaired',\n",
       " 'information',\n",
       " 'inspector',\n",
       " 'instruc',\n",
       " 'instruct',\n",
       " 'instruction',\n",
       " 'instructional',\n",
       " 'instructor',\n",
       " 'instrumental',\n",
       " 'inter',\n",
       " 'intern',\n",
       " 'interpreter',\n",
       " 'interrelated',\n",
       " 'intervention',\n",
       " 'intr',\n",
       " 'intrn',\n",
       " 'intructional',\n",
       " 'janitorial',\n",
       " 'japanese',\n",
       " 'job',\n",
       " 'jr',\n",
       " 'justice',\n",
       " 'kdg',\n",
       " 'kindergarten',\n",
       " 'labor',\n",
       " 'ladder',\n",
       " 'lang',\n",
       " 'large',\n",
       " 'latin',\n",
       " 'lead',\n",
       " 'leadman',\n",
       " 'learning',\n",
       " 'liason',\n",
       " 'librarian',\n",
       " 'licensed',\n",
       " 'lieutenant',\n",
       " 'life',\n",
       " 'literacy',\n",
       " 'lpn',\n",
       " 'm',\n",
       " 'mailroom',\n",
       " 'maintenance',\n",
       " 'man',\n",
       " 'manager',\n",
       " 'manager/liason',\n",
       " 'marketing',\n",
       " 'master',\n",
       " 'material',\n",
       " 'math',\n",
       " 'mechanic',\n",
       " 'med',\n",
       " 'medium',\n",
       " 'member',\n",
       " 'mental',\n",
       " 'mentally',\n",
       " 'mentor',\n",
       " 'mgr',\n",
       " 'mid',\n",
       " 'middle',\n",
       " 'mildly',\n",
       " 'minor',\n",
       " 'misc',\n",
       " 'mo',\n",
       " 'moderate',\n",
       " 'mon',\n",
       " 'mon-n/cert',\n",
       " 'monitor',\n",
       " 'month',\n",
       " 'music',\n",
       " 'n-faculty',\n",
       " 'n/c',\n",
       " 'nc',\n",
       " 'ncert',\n",
       " 'need',\n",
       " 'network',\n",
       " 'night',\n",
       " 'non',\n",
       " 'non-degreed',\n",
       " 'noncert',\n",
       " 'nurse',\n",
       " 'nutritionist',\n",
       " 'occupancy',\n",
       " 'occupation',\n",
       " 'occupational',\n",
       " 'occupational/physical',\n",
       " 'office',\n",
       " 'office/administrative',\n",
       " 'officer',\n",
       " 'oil',\n",
       " 'operating',\n",
       " 'operation',\n",
       " 'operator',\n",
       " 'optional',\n",
       " 'orthopedically',\n",
       " 'overtime',\n",
       " 'pac',\n",
       " 'painter',\n",
       " 'para',\n",
       " 'paraprof',\n",
       " 'paraprofessional',\n",
       " 'parent',\n",
       " 'parent/title',\n",
       " 'partially',\n",
       " 'patrol',\n",
       " 'pay',\n",
       " 'payroll',\n",
       " 'pe',\n",
       " 'phys',\n",
       " 'physic',\n",
       " 'physical',\n",
       " 'police',\n",
       " 'political',\n",
       " 'pos.cntrl',\n",
       " 'pre-k',\n",
       " 'pre-referral',\n",
       " 'prin',\n",
       " 'principal',\n",
       " 'print',\n",
       " 'proc',\n",
       " 'processing',\n",
       " 'production',\n",
       " 'prof',\n",
       " 'professional',\n",
       " 'professional-instructional',\n",
       " 'professional-other',\n",
       " 'program',\n",
       " 'programmer',\n",
       " 'psy',\n",
       " 'psych',\n",
       " 'psychological',\n",
       " 'psychologist',\n",
       " 'public',\n",
       " 'pump',\n",
       " 'purchased',\n",
       " 'radio',\n",
       " 'radio/tv',\n",
       " 'rate',\n",
       " 'reading',\n",
       " 'rec',\n",
       " 'receives',\n",
       " 'receiving',\n",
       " 'reg',\n",
       " 'registrar',\n",
       " 'regular',\n",
       " 'rental',\n",
       " 'rentals-land',\n",
       " 'repair',\n",
       " 'reqd',\n",
       " 'resource',\n",
       " 'retarded',\n",
       " 'retiree',\n",
       " 'rotc',\n",
       " 'router',\n",
       " 's.e',\n",
       " 'sac',\n",
       " 'sal',\n",
       " 'salaried',\n",
       " 'sale',\n",
       " 'sasi',\n",
       " 'sat',\n",
       " 'sch',\n",
       " 'sched',\n",
       " 'school',\n",
       " 'science',\n",
       " 'scrty',\n",
       " 'sec',\n",
       " 'secondary',\n",
       " 'secretary',\n",
       " 'security',\n",
       " 'sel-contained',\n",
       " 'self',\n",
       " 'senior',\n",
       " 'sergeant',\n",
       " 'service',\n",
       " 'severely',\n",
       " 'shift',\n",
       " 'shop',\n",
       " 'sighted',\n",
       " 'site',\n",
       " 'size',\n",
       " 'slp',\n",
       " 'small',\n",
       " 'social',\n",
       " 'solution',\n",
       " 'spanish',\n",
       " 'spec',\n",
       " 'special',\n",
       " 'specialist',\n",
       " 'speech',\n",
       " 'sr',\n",
       " 'sr.',\n",
       " 'staff',\n",
       " 'start',\n",
       " 'structure',\n",
       " 'structured',\n",
       " 'student',\n",
       " 'sub',\n",
       " 'subsidy',\n",
       " 'substitute',\n",
       " 'super',\n",
       " 'superintendent',\n",
       " 'supervisor',\n",
       " 'suppl',\n",
       " 'supply',\n",
       " 'support',\n",
       " 'supt',\n",
       " 'supv',\n",
       " 'ta',\n",
       " 'talented',\n",
       " 'tap',\n",
       " 'tchracct',\n",
       " 'teacher',\n",
       " 'team',\n",
       " 'tech',\n",
       " 'technical',\n",
       " 'technician',\n",
       " 'technology',\n",
       " 'telecommunication',\n",
       " 'telegram',\n",
       " 'telephone',\n",
       " 'teletype',\n",
       " 'temp',\n",
       " 'temp-summer',\n",
       " 'temporary',\n",
       " 'textbook',\n",
       " 'therapist',\n",
       " 'therapy',\n",
       " 'therp',\n",
       " 'time',\n",
       " 'timecard',\n",
       " 'trade',\n",
       " 'train',\n",
       " 'transfer',\n",
       " 'transition',\n",
       " 'transportation',\n",
       " 'travel',\n",
       " 'traveling',\n",
       " 'trip',\n",
       " 'truck',\n",
       " 'tutoring',\n",
       " 'undesignated',\n",
       " 'video',\n",
       " 'visual',\n",
       " 'voc',\n",
       " 'vocational',\n",
       " 'voe',\n",
       " 'w/doctorate',\n",
       " 'warehouse',\n",
       " 'web',\n",
       " 'wkr',\n",
       " 'worke',\n",
       " 'worker',\n",
       " 'wrk',\n",
       " 'wrong-used']"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vec_alphanumeric.get_feature_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "385"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(vec_alphanumeric.get_feature_names()) # vanilla version CountTokenizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "370"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(vec_alphanumeric.get_feature_names()) # Using stop words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "481"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(vec_alphanumeric.get_feature_names()) # Using LemmaTokenizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "list"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(vec_alphanumeric.get_feature_names())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "LABELS = labels\n",
    "NUMERIC_COLUMNS = numeric_columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In order to get a bag-of-words representation for all of the text data in our DataFrame, we must first convert the text data in each row of the DataFrame into a single string.\n",
    "\n",
    "We proceed to do so by grabbing all the data from columns which are not numeric or label data and then combine it into one string for each row to pass to the tokenizer. Note: above when we were looking at just one column 'Position_Extra', we automatically had one string per row. \n",
    "Now, we will create a bag of words for each row by joining the strings. \n",
    "\n",
    "We write a function for that."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Function',\n",
       " 'Use',\n",
       " 'Sharing',\n",
       " 'Reporting',\n",
       " 'Student_Type',\n",
       " 'Position_Type',\n",
       " 'Object_Type',\n",
       " 'Pre_K',\n",
       " 'Operating_Status']"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "LABELS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Function',\n",
       " 'Use',\n",
       " 'Sharing',\n",
       " 'Reporting',\n",
       " 'Student_Type',\n",
       " 'Position_Type',\n",
       " 'Object_Type',\n",
       " 'Pre_K',\n",
       " 'Operating_Status',\n",
       " 'Object_Description',\n",
       " 'Text_2',\n",
       " 'SubFund_Description',\n",
       " 'Job_Title_Description',\n",
       " 'Text_3',\n",
       " 'Text_4',\n",
       " 'Sub_Object_Description',\n",
       " 'Location_Description',\n",
       " 'FTE',\n",
       " 'Function_Description',\n",
       " 'Facility_or_Department',\n",
       " 'Position_Extra',\n",
       " 'Total',\n",
       " 'Program_Description',\n",
       " 'Fund_Description',\n",
       " 'Text_1']"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.columns.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "to_drop = NUMERIC_COLUMNS + LABELS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'FTE',\n",
       " 'Function',\n",
       " 'Object_Type',\n",
       " 'Operating_Status',\n",
       " 'Position_Type',\n",
       " 'Pre_K',\n",
       " 'Reporting',\n",
       " 'Sharing',\n",
       " 'Student_Type',\n",
       " 'Total',\n",
       " 'Use'}"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set(to_drop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'FTE',\n",
       " 'Function',\n",
       " 'Object_Type',\n",
       " 'Operating_Status',\n",
       " 'Position_Type',\n",
       " 'Pre_K',\n",
       " 'Reporting',\n",
       " 'Sharing',\n",
       " 'Student_Type',\n",
       " 'Total',\n",
       " 'Use'}"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set(to_drop) & set(df_train.columns.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def combine_text_columns(df, to_drop = NUMERIC_COLUMNS + LABELS):\n",
    "    \"\"\"converts all text in each row of data_frame df to a single vector\"\"\"\n",
    "    \n",
    "    #Drop non-text columns that are in df\n",
    "    to_drop = set(to_drop) & set(df.columns.tolist())\n",
    "    \n",
    "    text_data = df.drop(to_drop, axis = 1)\n",
    "    \n",
    "    # Replace nans with blanks\n",
    "    text_data.fillna('', inplace = True)\n",
    "    \n",
    "    # Join all text items in a row that have a space in between\n",
    "    \n",
    "    return text_data.apply(lambda x: ' '.join(x), axis = 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now you will use combine_text_columns() to convert all training text data in your DataFrame to a single vector that can be passed to the vectorizer object and made into a bag-of-words using the .fit_transform() method.\n",
    "\n",
    "We will use two ways of tokenizing. One using any non-whitespace characters as a token and other using only alphanumeric characters as token."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Create the basic token pattern\n",
    "TOKENS_BASIC = '\\\\S+(?=\\\\s+)'\n",
    "\n",
    "# Create the alphanumeric token pattern\n",
    "TOKENS_ALPHANUMERIC = '[A-Za-z0-9]+(?=\\\\s+)'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vec_basic = CountVectorizer(token_pattern=TOKENS_BASIC)\n",
    "vec_alphanumeric = CountVectorizer(token_pattern=TOKENS_ALPHANUMERIC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "text_vector = combine_text_columns(df_train, to_drop = NUMERIC_COLUMNS + LABELS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pandas.core.series.Series"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(text_vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "134338       Teacher-Elementary        KINDERGARTEN  KIN...\n",
       "206341    CONTRACTOR SERVICES BOND EXPENDITURES BUILDING...\n",
       "326408    Personal Services - Teachers   TCHER 2ND GRADE...\n",
       "364634    EMPLOYEE BENEFITS TEACHER SUBS GENERAL FUND Te...\n",
       "47683     TEACHER COVERAGE FOR TEACHER TEACHER SUBS GENE...\n",
       "dtype: object"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_vector.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(400277, 25)"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<400277x3284 sparse matrix of type '<class 'numpy.int64'>'\n",
       "\twith 5660399 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vec_basic.fit_transform(text_vector)\n",
    "vec_alphanumeric.fit_transform(text_vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['\"(year',\n",
       " '\"building',\n",
       " '\"guidance,',\n",
       " '\"human',\n",
       " '\"maintenance,',\n",
       " '\"multilingual',\n",
       " '\"performing',\n",
       " '\"software,',\n",
       " '\"technology',\n",
       " '\"title',\n",
       " '%',\n",
       " '&',\n",
       " '&materials',\n",
       " '&program',\n",
       " '&wildlife',\n",
       " '(',\n",
       " '(4-5)',\n",
       " '(6-8)',\n",
       " '(720)',\n",
       " '(9-',\n",
       " '(9-12)',\n",
       " '(admin',\n",
       " '(afterschool)',\n",
       " '(appl',\n",
       " '(arra)',\n",
       " '(athletic',\n",
       " '(attendance)',\n",
       " '(aut/md)',\n",
       " '(autsm)',\n",
       " '(avid)',\n",
       " '(bilingual)',\n",
       " '(blank)',\n",
       " '(boys)',\n",
       " '(byot)',\n",
       " '(career',\n",
       " '(cbla)',\n",
       " '(cc)',\n",
       " '(cd)',\n",
       " '(cdc-a',\n",
       " '(cdc-a)',\n",
       " '(cdc-a/elem)',\n",
       " '(cdc-a/high)',\n",
       " '(cdc-a/mid)',\n",
       " '(cdc-a/special)',\n",
       " '(cdc-elem)',\n",
       " '(cdc-middle)',\n",
       " '(cdc-secondary)',\n",
       " '(cdc/elem)',\n",
       " '(cdc/high)',\n",
       " '(cdc/mid)',\n",
       " '(cera)',\n",
       " '(cert)',\n",
       " '(child',\n",
       " '(clk&rtrs',\n",
       " '(computer',\n",
       " '(constrct)',\n",
       " '(core)',\n",
       " '(def',\n",
       " '(ed)',\n",
       " '(eg',\n",
       " '(eg)',\n",
       " '(el',\n",
       " '(el)',\n",
       " '(elem',\n",
       " '(elem)',\n",
       " '(eng',\n",
       " '(english)',\n",
       " '(esol)',\n",
       " '(et)',\n",
       " '(ett)',\n",
       " '(excl',\n",
       " '(expenditures',\n",
       " '(external)',\n",
       " '(fam/comm)',\n",
       " '(feb-jun',\n",
       " '(fed',\n",
       " '(fee)',\n",
       " '(fefp',\n",
       " '(gardener)',\n",
       " '(gift)',\n",
       " '(girls)',\n",
       " '(gr',\n",
       " '(grade',\n",
       " '(grades',\n",
       " '(head)',\n",
       " '(headstart)',\n",
       " '(hh)',\n",
       " '(high)',\n",
       " '(host)',\n",
       " '(hs)',\n",
       " '(ii)',\n",
       " '(impact)',\n",
       " '(including',\n",
       " '(interim)',\n",
       " '(iss/high)',\n",
       " '(iss/mid)',\n",
       " '(itemga)',\n",
       " '(job',\n",
       " '(jul-aug',\n",
       " '(k-12)',\n",
       " '(language',\n",
       " '(librarian)',\n",
       " '(licensed',\n",
       " '(lottery)',\n",
       " '(magnet-elem)',\n",
       " '(magnet/elem)',\n",
       " '(maint.',\n",
       " '(md)',\n",
       " '(me)',\n",
       " '(mf)',\n",
       " '(mid)',\n",
       " '(middle',\n",
       " '(middle)',\n",
       " '(no',\n",
       " '(non-local)',\n",
       " '(not',\n",
       " '(office',\n",
       " '(oh)',\n",
       " '(oil)',\n",
       " '(one-on-one)',\n",
       " '(only',\n",
       " '(ose)',\n",
       " '(ot)',\n",
       " '(other',\n",
       " '(participatory',\n",
       " '(perishable',\n",
       " '(phy',\n",
       " '(pool',\n",
       " '(pool)',\n",
       " '(position',\n",
       " '(prevent',\n",
       " '(pt)',\n",
       " '(recycling)',\n",
       " '(rent,',\n",
       " '(resource/elem)',\n",
       " '(resource/high)',\n",
       " '(resource/mid)',\n",
       " '(satellite/elem)',\n",
       " '(satellite/high)',\n",
       " '(satellite/mid)',\n",
       " '(satellite/special)',\n",
       " '(scc)tech',\n",
       " '(school',\n",
       " '(school)',\n",
       " '(sec)',\n",
       " '(secondary)',\n",
       " '(security)',\n",
       " '(slp)',\n",
       " '(sp.ed.)',\n",
       " '(spec',\n",
       " '(speced',\n",
       " '(special',\n",
       " '(special)',\n",
       " '(state',\n",
       " '(summ',\n",
       " '(summer',\n",
       " '(summer)',\n",
       " '(supplemented)',\n",
       " '(tanf)',\n",
       " '(tap)',\n",
       " '(tif)',\n",
       " '(trans)',\n",
       " '(vi)',\n",
       " '(vis)',\n",
       " '(was',\n",
       " '(zz)',\n",
       " ')',\n",
       " '*',\n",
       " '*******suspense*******',\n",
       " '+',\n",
       " ',',\n",
       " '-',\n",
       " '--',\n",
       " '-2nd',\n",
       " '-aba',\n",
       " '-academic',\n",
       " '-accelerated',\n",
       " '-arra',\n",
       " '-beyond',\n",
       " '-ceo',\n",
       " '-crisis',\n",
       " '-emergency',\n",
       " '-ends',\n",
       " '-fam',\n",
       " '-gender',\n",
       " '-host',\n",
       " '-humanware',\n",
       " '-non-instruction',\n",
       " '-orl',\n",
       " '-pre-k',\n",
       " '-rsb',\n",
       " '-sch',\n",
       " '-software',\n",
       " '-strategic',\n",
       " '-yr',\n",
       " '/',\n",
       " '/fte',\n",
       " '/or',\n",
       " '1st',\n",
       " '2nd',\n",
       " '3rd',\n",
       " '4th',\n",
       " '5/6',\n",
       " '56',\n",
       " '5th',\n",
       " '6,7,8',\n",
       " '6-8',\n",
       " '6-8)',\n",
       " '6-8,',\n",
       " '6th',\n",
       " '7-12',\n",
       " '84.00a',\n",
       " '9-12',\n",
       " '9th',\n",
       " '>',\n",
       " '@',\n",
       " 'a',\n",
       " 'a,',\n",
       " 'a-teacher',\n",
       " 'a:',\n",
       " 'ab',\n",
       " 'aba',\n",
       " 'abatement',\n",
       " 'absence',\n",
       " 'ac',\n",
       " 'acad',\n",
       " 'academic',\n",
       " 'academic)',\n",
       " 'academically',\n",
       " 'academy',\n",
       " 'acadmey',\n",
       " 'accelerated',\n",
       " 'access',\n",
       " 'access(constrct',\n",
       " 'accessories',\n",
       " 'accident',\n",
       " 'accnts',\n",
       " 'accompanists',\n",
       " 'accoun',\n",
       " 'account',\n",
       " 'account.',\n",
       " 'accountability',\n",
       " 'accountability,',\n",
       " 'accountability-research-eval',\n",
       " 'accountant',\n",
       " 'accountant,',\n",
       " 'accountants',\n",
       " 'accounting',\n",
       " 'accounts',\n",
       " 'accounts-activity',\n",
       " 'accounts-governmental',\n",
       " 'acct',\n",
       " 'acct.',\n",
       " 'acctblty',\n",
       " 'achiev',\n",
       " 'achieve',\n",
       " 'achievment',\n",
       " 'achv',\n",
       " 'acq',\n",
       " 'acquis',\n",
       " 'acquisition',\n",
       " 'acquisition,',\n",
       " 'acquisition-current',\n",
       " 'acquisitions',\n",
       " 'act',\n",
       " 'act)',\n",
       " 'acte',\n",
       " 'acting',\n",
       " 'action',\n",
       " 'active',\n",
       " 'activi',\n",
       " 'activiti',\n",
       " 'activities',\n",
       " 'activity',\n",
       " 'activity)',\n",
       " 'activties',\n",
       " 'actually',\n",
       " 'ad',\n",
       " 'ada',\n",
       " 'adapted',\n",
       " 'adaptive',\n",
       " \"add'l\",\n",
       " 'addit',\n",
       " 'addition',\n",
       " 'additional',\n",
       " 'additional/extra',\n",
       " 'addl',\n",
       " 'adie',\n",
       " 'aditional',\n",
       " 'adjustme',\n",
       " 'adjustment',\n",
       " 'adjustments',\n",
       " 'adl',\n",
       " 'adm',\n",
       " 'adm/prod.leader',\n",
       " 'adm/prof',\n",
       " 'admin',\n",
       " 'admin.',\n",
       " 'admin.dept.',\n",
       " 'adminis',\n",
       " 'administ.',\n",
       " 'administrat',\n",
       " 'administration',\n",
       " 'administration\"',\n",
       " 'administration*',\n",
       " 'administrative',\n",
       " 'administrative/supervisors*',\n",
       " 'administrator',\n",
       " 'administrator,',\n",
       " 'administrators',\n",
       " 'adminsitrative',\n",
       " 'adminstrative',\n",
       " 'adminstriative',\n",
       " 'admissions',\n",
       " 'admn',\n",
       " 'admn.',\n",
       " 'adopt',\n",
       " 'adopted',\n",
       " 'adult',\n",
       " 'adult/continuing',\n",
       " 'adults',\n",
       " 'advanc',\n",
       " 'advance',\n",
       " 'advanced',\n",
       " 'advancement',\n",
       " 'advances',\n",
       " 'advances-out',\n",
       " 'advaned',\n",
       " 'advantage',\n",
       " 'advertisements',\n",
       " 'advertising',\n",
       " 'advisor',\n",
       " 'advisor,',\n",
       " 'advncmnt',\n",
       " 'advocate',\n",
       " 'ae',\n",
       " 'aefla',\n",
       " 'aetna',\n",
       " 'affair',\n",
       " 'affairs',\n",
       " 'aft',\n",
       " 'after',\n",
       " 'afterschool',\n",
       " 'against',\n",
       " 'agencies',\n",
       " 'agency',\n",
       " 'agenda',\n",
       " 'agent',\n",
       " 'agents',\n",
       " 'ages',\n",
       " 'agreement',\n",
       " 'agreements',\n",
       " 'agricultural',\n",
       " 'agriculture',\n",
       " 'agriculture/envrmntl',\n",
       " 'agriscience',\n",
       " 'aha',\n",
       " 'aid',\n",
       " 'aid,',\n",
       " 'aide',\n",
       " 'aide(ded)',\n",
       " 'aide,',\n",
       " 'aide,60',\n",
       " 'aide,60hrs',\n",
       " 'aide,70',\n",
       " 'aide,70hrs',\n",
       " 'aide,70hrs.',\n",
       " 'aide,70hs',\n",
       " 'aide,80',\n",
       " 'aide,80hrs.',\n",
       " 'aide,80hs',\n",
       " 'aide,bilingua',\n",
       " 'aide,computer',\n",
       " 'aide,ded,70hr',\n",
       " 'aide,headstar',\n",
       " 'aide,headstart',\n",
       " 'aide,library',\n",
       " 'aide,pre-k',\n",
       " 'aide,regular',\n",
       " 'aide,spec.ed',\n",
       " 'aide,sped',\n",
       " 'aide,sped,70h',\n",
       " 'aide,wae',\n",
       " 'aided',\n",
       " 'aides',\n",
       " 'aides,',\n",
       " 'aids',\n",
       " 'aids-lunch',\n",
       " 'air',\n",
       " 'aircraft',\n",
       " 'alao',\n",
       " 'alarm',\n",
       " 'alcolhol',\n",
       " 'aligning',\n",
       " 'all',\n",
       " 'alliance',\n",
       " 'alliance-swap',\n",
       " 'alloc',\n",
       " 'allocation',\n",
       " 'allocation-hst',\n",
       " 'allocations',\n",
       " 'allotment',\n",
       " 'allowance',\n",
       " 'allowances',\n",
       " 'allstate',\n",
       " 'alpha',\n",
       " 'alrm',\n",
       " 'alrms,intrcom-host',\n",
       " 'alt',\n",
       " 'alt/remodeling',\n",
       " 'alter',\n",
       " 'alternative',\n",
       " 'alumni',\n",
       " 'am',\n",
       " 'ambassador',\n",
       " 'amd-fs',\n",
       " 'amendment',\n",
       " 'amer.old',\n",
       " 'amer/trw',\n",
       " 'america',\n",
       " 'american',\n",
       " 'americorp',\n",
       " 'americorps',\n",
       " 'ameritech',\n",
       " 'an',\n",
       " 'ana',\n",
       " 'analysis',\n",
       " 'analyst',\n",
       " 'analyst,',\n",
       " 'analyst-',\n",
       " 'ancora',\n",
       " 'and',\n",
       " 'anlayst',\n",
       " 'anlyt',\n",
       " 'annex',\n",
       " 'annual',\n",
       " 'annuity',\n",
       " 'anonymous',\n",
       " 'another',\n",
       " 'answer',\n",
       " 'anticipated',\n",
       " 'any',\n",
       " 'aob',\n",
       " 'ap',\n",
       " 'ap,',\n",
       " 'app',\n",
       " 'app/non-instru',\n",
       " 'appeals',\n",
       " 'appl',\n",
       " 'apple',\n",
       " 'appliance',\n",
       " 'application',\n",
       " 'applications',\n",
       " 'appraisal',\n",
       " 'apprentice',\n",
       " 'apprenticeship',\n",
       " 'apprenticeships',\n",
       " 'approp',\n",
       " 'appropriation',\n",
       " 'approval',\n",
       " 'apps',\n",
       " 'aquatic',\n",
       " 'arbitrage',\n",
       " 'arch',\n",
       " 'archaelogical',\n",
       " 'architect',\n",
       " 'architectural&engineering',\n",
       " 'architecture',\n",
       " 'ard',\n",
       " 'area',\n",
       " 'army',\n",
       " 'around',\n",
       " 'arra',\n",
       " 'arra(stars)',\n",
       " 'arra-homeless',\n",
       " 'arra-i-a',\n",
       " 'arra-nat',\n",
       " 'arra-nonpublic',\n",
       " 'arra-salary',\n",
       " 'arra-sw',\n",
       " 'arra-t-i',\n",
       " 'arra-title',\n",
       " 'art',\n",
       " 'art-elem',\n",
       " 'art-high',\n",
       " 'art-middle',\n",
       " 'articulated',\n",
       " 'artist',\n",
       " 'arts',\n",
       " 'arts,other',\n",
       " 'arts-high',\n",
       " 'arts-middle',\n",
       " 'arts/culture',\n",
       " 'arts/tech',\n",
       " 'arts/technology',\n",
       " 'asbestos',\n",
       " 'asd',\n",
       " 'asfepa-',\n",
       " 'asia',\n",
       " 'aside',\n",
       " 'ass.',\n",
       " 'assembler',\n",
       " 'assesment',\n",
       " 'assess',\n",
       " 'assess.',\n",
       " 'assessment',\n",
       " 'assessments',\n",
       " 'asset',\n",
       " 'assets',\n",
       " 'assets,',\n",
       " 'assgmt',\n",
       " 'assgn',\n",
       " 'assgnmt',\n",
       " 'assi',\n",
       " 'assign',\n",
       " 'assign,',\n",
       " 'assign-sh',\n",
       " 'assign/attendance/re',\n",
       " 'assignment',\n",
       " 'assignments',\n",
       " 'assis',\n",
       " 'assist',\n",
       " 'assist(cdc/special',\n",
       " 'assist(satellite/special)',\n",
       " 'assista',\n",
       " 'assistan',\n",
       " 'assistance',\n",
       " 'assistane',\n",
       " 'assistant',\n",
       " 'assistant,',\n",
       " 'assistant-elem',\n",
       " 'assistant-ell',\n",
       " 'assistant-esol',\n",
       " 'assistant-high',\n",
       " 'assistant-pk',\n",
       " 'assistant-rotc',\n",
       " 'assistant-title',\n",
       " 'assistants',\n",
       " 'assistive',\n",
       " 'assitant',\n",
       " 'assmt',\n",
       " 'assmt/tst',\n",
       " 'assn',\n",
       " 'assoc',\n",
       " 'assoc,',\n",
       " 'assoc.',\n",
       " 'associate',\n",
       " 'associate,project',\n",
       " 'associated',\n",
       " 'asst',\n",
       " 'asst,',\n",
       " 'asst-boys',\n",
       " 'asst-girls',\n",
       " 'asst-sp',\n",
       " 'asst.',\n",
       " 'asst/clerk',\n",
       " 'asst/exec',\n",
       " 'asst/records',\n",
       " 'assur',\n",
       " 'assurance',\n",
       " 'asthma',\n",
       " 'at',\n",
       " 'ath',\n",
       " 'athletic',\n",
       " 'athletic/academic',\n",
       " 'athletics',\n",
       " 'athletics/sports',\n",
       " 'atrt',\n",
       " 'atsp',\n",
       " 'attendance',\n",
       " 'attendant',\n",
       " 'attendence',\n",
       " 'attn',\n",
       " 'attorney',\n",
       " 'attorneys',\n",
       " 'attorneys,legal',\n",
       " 'atwll',\n",
       " 'audio',\n",
       " 'audio/visual',\n",
       " 'audiologist',\n",
       " 'audiology',\n",
       " 'audiovisual',\n",
       " 'audit',\n",
       " 'auditing',\n",
       " 'auditor',\n",
       " 'auditor/accountant',\n",
       " 'auditory',\n",
       " 'august',\n",
       " 'aut/md',\n",
       " 'autism',\n",
       " 'autistic',\n",
       " 'auto',\n",
       " 'auto:hs',\n",
       " 'automation',\n",
       " 'automotive',\n",
       " 'autonamy',\n",
       " 'autonomous',\n",
       " 'aux',\n",
       " 'auxiliary',\n",
       " 'av',\n",
       " 'av/archivist',\n",
       " 'av/pc',\n",
       " 'avenue',\n",
       " 'avg',\n",
       " 'aviation',\n",
       " 'avid',\n",
       " 'award',\n",
       " 'awards',\n",
       " 'awards/prizes',\n",
       " 'ayp',\n",
       " 'b',\n",
       " 'b-',\n",
       " 'b-main',\n",
       " 'b-team',\n",
       " 'b-team-bo',\n",
       " 'b-team-gi',\n",
       " 'b:',\n",
       " 'ba',\n",
       " 'baccalaur',\n",
       " 'baccalaurate',\n",
       " 'bachelor',\n",
       " 'back',\n",
       " 'background',\n",
       " 'backpack',\n",
       " 'baker',\n",
       " 'bakery',\n",
       " 'band',\n",
       " 'bank',\n",
       " 'banking',\n",
       " 'banquets',\n",
       " 'barbering',\n",
       " 'barrier',\n",
       " 'base/sftball',\n",
       " 'baseball',\n",
       " 'based',\n",
       " 'basic',\n",
       " 'basis',\n",
       " 'basketball',\n",
       " 'bbb',\n",
       " 'bckflw',\n",
       " 'bd',\n",
       " 'bdgt',\n",
       " 'be',\n",
       " 'beat',\n",
       " 'beginning',\n",
       " 'behav',\n",
       " 'behav.',\n",
       " 'behavior',\n",
       " 'behavioral',\n",
       " 'behaviorial/interv',\n",
       " 'behavorial',\n",
       " 'believers',\n",
       " 'bell',\n",
       " 'benchmark',\n",
       " 'benefit',\n",
       " 'benefits',\n",
       " 'benefits)',\n",
       " 'benefits,',\n",
       " 'benefits/wc',\n",
       " 'best',\n",
       " 'bga',\n",
       " 'bga-gesp',\n",
       " 'bgt',\n",
       " 'bi',\n",
       " 'bible',\n",
       " 'bicycle',\n",
       " 'bil',\n",
       " 'biling',\n",
       " 'bilingual',\n",
       " 'bilingual,',\n",
       " 'bilingual/multicultural',\n",
       " 'bilinguial',\n",
       " 'billing',\n",
       " 'bilng',\n",
       " 'bilngl',\n",
       " 'binding',\n",
       " 'binding,',\n",
       " 'binligual',\n",
       " 'bio/chem',\n",
       " 'biological',\n",
       " 'biology',\n",
       " 'bird',\n",
       " 'birth',\n",
       " 'birth/kind',\n",
       " 'bks',\n",
       " 'bks-hst',\n",
       " 'black',\n",
       " 'bldg',\n",
       " 'bldg.&',\n",
       " 'bldgs',\n",
       " 'blind',\n",
       " 'blngl',\n",
       " 'block',\n",
       " 'bnc-supervisor',\n",
       " 'board',\n",
       " 'body',\n",
       " 'body-sound',\n",
       " 'boiler',\n",
       " 'boiler/combustion',\n",
       " 'boilermaker',\n",
       " 'boilers',\n",
       " 'bolton',\n",
       " 'bond',\n",
       " 'bonding',\n",
       " 'bonds',\n",
       " 'bonus',\n",
       " 'bonuses',\n",
       " 'book',\n",
       " 'bookkeeper',\n",
       " 'bookkeepers',\n",
       " 'books',\n",
       " 'books,',\n",
       " 'boost',\n",
       " 'boot',\n",
       " 'bottled',\n",
       " 'bowl-curriculum',\n",
       " 'boys',\n",
       " 'bp',\n",
       " 'braille',\n",
       " 'brandmuscle-adobe',\n",
       " 'brd',\n",
       " 'breaker',\n",
       " 'breakfast',\n",
       " 'brick',\n",
       " 'bricklayer',\n",
       " 'bricks',\n",
       " 'bridge',\n",
       " 'bright',\n",
       " 'brkfast',\n",
       " 'broad',\n",
       " 'broadcast',\n",
       " 'broadcasting',\n",
       " 'broken',\n",
       " 'brothers',\n",
       " 'bsi',\n",
       " 'budg',\n",
       " 'budget',\n",
       " 'budget/finance',\n",
       " 'budgetary',\n",
       " 'budgeting',\n",
       " 'budgets',\n",
       " 'budgets/schools',\n",
       " 'buhrer',\n",
       " 'building',\n",
       " 'building/property',\n",
       " 'buildings',\n",
       " 'buisness',\n",
       " 'bulk',\n",
       " 'bump',\n",
       " 'bunch',\n",
       " 'bureau',\n",
       " 'bus',\n",
       " 'bus.&',\n",
       " 'bus/lbr/univ/cult',\n",
       " 'buses',\n",
       " 'busine',\n",
       " 'business',\n",
       " 'business,',\n",
       " 'business/admin',\n",
       " 'buy',\n",
       " 'buyer',\n",
       " 'by',\n",
       " 'c',\n",
       " 'c&i',\n",
       " 'c.o.p.',\n",
       " 'c/o-early',\n",
       " 'c/o-summer',\n",
       " 'cabinet',\n",
       " 'cabinetmaker',\n",
       " 'cable',\n",
       " 'cad',\n",
       " 'cadre',\n",
       " 'cafe',\n",
       " 'cafeteria',\n",
       " 'camera',\n",
       " 'camp',\n",
       " 'camp-summer',\n",
       " 'campbell',\n",
       " 'camps',\n",
       " 'campus',\n",
       " 'candies',\n",
       " 'candle',\n",
       " 'candy',\n",
       " 'cao',\n",
       " 'cap',\n",
       " 'capacity',\n",
       " 'cape-undesignated',\n",
       " 'capital',\n",
       " 'capitalized',\n",
       " 'captain',\n",
       " 'car-tech',\n",
       " 'card',\n",
       " 'cards',\n",
       " 'care',\n",
       " 'careeer',\n",
       " 'career',\n",
       " 'career/technology',\n",
       " 'caretaker',\n",
       " 'carl',\n",
       " 'carpenter',\n",
       " 'carpentry',\n",
       " 'carpentry/cabinet',\n",
       " 'carpet',\n",
       " 'carreer',\n",
       " 'carrier',\n",
       " 'carryforward',\n",
       " 'carryover',\n",
       " 'carryover-summer',\n",
       " 'case',\n",
       " 'cash',\n",
       " 'cashier',\n",
       " 'catalyst',\n",
       " 'category',\n",
       " 'catholic',\n",
       " 'cc/oh',\n",
       " 'cclc',\n",
       " 'ccoach',\n",
       " 'cdb',\n",
       " 'cde/qwest',\n",
       " 'ce',\n",
       " 'cec',\n",
       " 'cec-autobody',\n",
       " 'cec-automechanics',\n",
       " 'cec-child',\n",
       " 'cec-restaurant',\n",
       " 'ceis',\n",
       " 'cela',\n",
       " 'celeb',\n",
       " 'cen',\n",
       " 'cent',\n",
       " 'center',\n",
       " 'center-rec',\n",
       " 'centers',\n",
       " 'central',\n",
       " 'century',\n",
       " 'century/',\n",
       " 'ceo',\n",
       " 'cert',\n",
       " 'cert.',\n",
       " 'certificated',\n",
       " 'certification',\n",
       " 'certifications-life',\n",
       " 'certified',\n",
       " 'certifiedaddl',\n",
       " 'cf',\n",
       " 'cfda',\n",
       " 'cfo',\n",
       " 'ch',\n",
       " 'cha',\n",
       " 'chai',\n",
       " 'chair',\n",
       " 'chair.',\n",
       " 'challenge',\n",
       " 'challenge/fy',\n",
       " 'champions',\n",
       " 'chance',\n",
       " 'chancellor',\n",
       " 'chanclr',\n",
       " 'change',\n",
       " 'chapter',\n",
       " 'character',\n",
       " 'charge',\n",
       " 'charge/reimbur',\n",
       " 'charges',\n",
       " 'charitable',\n",
       " 'charities',\n",
       " 'charity',\n",
       " 'charity-',\n",
       " 'charity-imp',\n",
       " 'charter',\n",
       " 'charters',\n",
       " 'charting',\n",
       " 'chdrn',\n",
       " 'che',\n",
       " 'cheerlea',\n",
       " 'cheerleader',\n",
       " 'cheerleaders',\n",
       " 'cheerleading',\n",
       " 'chef',\n",
       " 'chemistry',\n",
       " 'chief',\n",
       " 'chief,',\n",
       " 'chief,hmn',\n",
       " 'chief,organizational',\n",
       " 'chief-facilities/operations',\n",
       " 'child',\n",
       " 'childcare',\n",
       " 'childhd',\n",
       " 'childhood',\n",
       " 'childn/youth',\n",
       " 'children',\n",
       " \"children's\",\n",
       " 'children/targeted',\n",
       " 'chiller/absorber',\n",
       " 'chinese',\n",
       " 'chllnge',\n",
       " 'choice',\n",
       " 'choice-county',\n",
       " 'choir',\n",
       " 'choir,',\n",
       " 'choral',\n",
       " 'chorus',\n",
       " 'chosen',\n",
       " 'christian',\n",
       " 'chrtr',\n",
       " 'church',\n",
       " 'cie',\n",
       " 'cisco',\n",
       " 'city',\n",
       " 'city-rsb',\n",
       " 'city-rsb-fy',\n",
       " 'city:',\n",
       " 'civics',\n",
       " 'claiming',\n",
       " 'claims',\n",
       " 'class',\n",
       " 'class-size',\n",
       " 'classified',\n",
       " 'classroom',\n",
       " 'classroon',\n",
       " 'clean',\n",
       " 'cleaner',\n",
       " 'cleaners',\n",
       " 'cleaning',\n",
       " 'cleaning/chemical',\n",
       " 'clerical',\n",
       " 'clerical,',\n",
       " 'clerical,area',\n",
       " 'clerical,district',\n",
       " 'clerical,warehouse/receiving',\n",
       " 'clerical-summer',\n",
       " 'clerical/bookkeeper',\n",
       " 'clerical/tchr',\n",
       " 'clerk',\n",
       " 'clerk,',\n",
       " 'clerk,data',\n",
       " 'clerk,library',\n",
       " 'clerk,non-typing',\n",
       " 'clerk,office',\n",
       " 'clerk,typing',\n",
       " 'clerk/data',\n",
       " 'client',\n",
       " 'climate',\n",
       " 'climbing',\n",
       " 'clinic',\n",
       " 'clinic-math',\n",
       " 'clinical',\n",
       " 'clk',\n",
       " 'clk/fire',\n",
       " 'closed',\n",
       " 'closeures',\n",
       " 'closing',\n",
       " 'clothing',\n",
       " 'clout',\n",
       " 'club',\n",
       " 'cluster',\n",
       " 'cms',\n",
       " 'cmsd',\n",
       " 'cnflct',\n",
       " 'cnslg',\n",
       " 'cntr',\n",
       " 'cntrl',\n",
       " 'cntry',\n",
       " 'co',\n",
       " 'co-curricular',\n",
       " 'co-dispatcher',\n",
       " 'co-praxis',\n",
       " 'co-tchr',\n",
       " 'coa',\n",
       " 'coac',\n",
       " 'coach',\n",
       " 'coach,',\n",
       " 'coach/specialist',\n",
       " 'coaches',\n",
       " 'coaching',\n",
       " 'coalit',\n",
       " 'coalition',\n",
       " 'coappp',\n",
       " 'coatings',\n",
       " 'cocurricular',\n",
       " 'cocurriculum/extra-curricular',\n",
       " 'code',\n",
       " 'cogat-gt',\n",
       " 'cohart',\n",
       " 'cohort',\n",
       " 'coll',\n",
       " 'collabo',\n",
       " 'collabor',\n",
       " 'collaboratives',\n",
       " 'collection',\n",
       " 'collections',\n",
       " 'college',\n",
       " 'college/schools',\n",
       " 'collegiate',\n",
       " 'colleyball',\n",
       " 'collision',\n",
       " 'colorado',\n",
       " 'com',\n",
       " 'combination',\n",
       " 'comes',\n",
       " ...]"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vec_basic.get_feature_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['00a',\n",
       " '12',\n",
       " '1st',\n",
       " '2nd',\n",
       " '3rd',\n",
       " '4th',\n",
       " '5',\n",
       " '56',\n",
       " '5th',\n",
       " '6',\n",
       " '60',\n",
       " '60hrs',\n",
       " '6th',\n",
       " '70',\n",
       " '70h',\n",
       " '70hr',\n",
       " '70hrs',\n",
       " '70hs',\n",
       " '8',\n",
       " '80',\n",
       " '80hs',\n",
       " '9th',\n",
       " 'a',\n",
       " 'aaps',\n",
       " 'ab',\n",
       " 'aba',\n",
       " 'abatement',\n",
       " 'absence',\n",
       " 'absorber',\n",
       " 'ac',\n",
       " 'acad',\n",
       " 'academ',\n",
       " 'academic',\n",
       " 'academically',\n",
       " 'academy',\n",
       " 'acadmey',\n",
       " 'accelerated',\n",
       " 'access',\n",
       " 'accessories',\n",
       " 'accident',\n",
       " 'accnts',\n",
       " 'accompanists',\n",
       " 'accoun',\n",
       " 'account',\n",
       " 'accountability',\n",
       " 'accountant',\n",
       " 'accountants',\n",
       " 'accounting',\n",
       " 'accounts',\n",
       " 'acct',\n",
       " 'acctblty',\n",
       " 'achiev',\n",
       " 'achieve',\n",
       " 'achievment',\n",
       " 'achv',\n",
       " 'acq',\n",
       " 'acquis',\n",
       " 'acquisition',\n",
       " 'acquisitions',\n",
       " 'act',\n",
       " 'acte',\n",
       " 'acting',\n",
       " 'action',\n",
       " 'active',\n",
       " 'activi',\n",
       " 'activiti',\n",
       " 'activities',\n",
       " 'activity',\n",
       " 'activties',\n",
       " 'actually',\n",
       " 'actuary',\n",
       " 'ad',\n",
       " 'ada',\n",
       " 'adapted',\n",
       " 'adaptive',\n",
       " 'addit',\n",
       " 'addition',\n",
       " 'additional',\n",
       " 'addl',\n",
       " 'addtn',\n",
       " 'adie',\n",
       " 'aditional',\n",
       " 'adjustme',\n",
       " 'adjustment',\n",
       " 'adjustments',\n",
       " 'adl',\n",
       " 'adm',\n",
       " 'admin',\n",
       " 'adminis',\n",
       " 'administrat',\n",
       " 'administration',\n",
       " 'administrative',\n",
       " 'administrator',\n",
       " 'administrators',\n",
       " 'adminsitrative',\n",
       " 'adminstrative',\n",
       " 'adminstriative',\n",
       " 'admissions',\n",
       " 'admn',\n",
       " 'adobe',\n",
       " 'adopt',\n",
       " 'adopted',\n",
       " 'adult',\n",
       " 'adults',\n",
       " 'advanc',\n",
       " 'advance',\n",
       " 'advanced',\n",
       " 'advancement',\n",
       " 'advances',\n",
       " 'advaned',\n",
       " 'advantage',\n",
       " 'advertisements',\n",
       " 'advertising',\n",
       " 'advisor',\n",
       " 'advncmnt',\n",
       " 'advocate',\n",
       " 'ae',\n",
       " 'aefla',\n",
       " 'aetna',\n",
       " 'affair',\n",
       " 'affairs',\n",
       " 'aft',\n",
       " 'after',\n",
       " 'afterschool',\n",
       " 'against',\n",
       " 'agencies',\n",
       " 'agency',\n",
       " 'agenda',\n",
       " 'agent',\n",
       " 'agents',\n",
       " 'ages',\n",
       " 'agreement',\n",
       " 'agreements',\n",
       " 'agricultural',\n",
       " 'agriculture',\n",
       " 'agriscience',\n",
       " 'aha',\n",
       " 'aid',\n",
       " 'aide',\n",
       " 'aided',\n",
       " 'aides',\n",
       " 'aids',\n",
       " 'air',\n",
       " 'aircraft',\n",
       " 'alao',\n",
       " 'alarm',\n",
       " 'alcolhol',\n",
       " 'aligning',\n",
       " 'all',\n",
       " 'allcity',\n",
       " 'alliance',\n",
       " 'alloc',\n",
       " 'allocation',\n",
       " 'allocations',\n",
       " 'allotment',\n",
       " 'allowance',\n",
       " 'allowances',\n",
       " 'allstate',\n",
       " 'alpha',\n",
       " 'alrm',\n",
       " 'alt',\n",
       " 'alter',\n",
       " 'alternative',\n",
       " 'alumni',\n",
       " 'am',\n",
       " 'ambassador',\n",
       " 'amendment',\n",
       " 'america',\n",
       " 'american',\n",
       " 'americorp',\n",
       " 'americorps',\n",
       " 'ameritech',\n",
       " 'an',\n",
       " 'ana',\n",
       " 'analysis',\n",
       " 'analyst',\n",
       " 'ancora',\n",
       " 'and',\n",
       " 'animal',\n",
       " 'anlayst',\n",
       " 'anlyt',\n",
       " 'annex',\n",
       " 'annual',\n",
       " 'annuity',\n",
       " 'anonymous',\n",
       " 'another',\n",
       " 'anschut',\n",
       " 'answer',\n",
       " 'anticipated',\n",
       " 'any',\n",
       " 'aob',\n",
       " 'ap',\n",
       " 'app',\n",
       " 'appeals',\n",
       " 'appl',\n",
       " 'apple',\n",
       " 'appliance',\n",
       " 'application',\n",
       " 'applications',\n",
       " 'appraisal',\n",
       " 'apprec',\n",
       " 'apprentice',\n",
       " 'apprenticeship',\n",
       " 'apprenticeships',\n",
       " 'approp',\n",
       " 'appropriation',\n",
       " 'approval',\n",
       " 'apps',\n",
       " 'aquatic',\n",
       " 'arbitrage',\n",
       " 'arboreal',\n",
       " 'arch',\n",
       " 'archaelogical',\n",
       " 'architect',\n",
       " 'architecture',\n",
       " 'archivist',\n",
       " 'ard',\n",
       " 'area',\n",
       " 'army',\n",
       " 'around',\n",
       " 'arra',\n",
       " 'art',\n",
       " 'articulated',\n",
       " 'artist',\n",
       " 'arts',\n",
       " 'asbestos',\n",
       " 'asd',\n",
       " 'asia',\n",
       " 'aside',\n",
       " 'assembler',\n",
       " 'assesment',\n",
       " 'assess',\n",
       " 'assessment',\n",
       " 'assessments',\n",
       " 'asset',\n",
       " 'assets',\n",
       " 'assgmt',\n",
       " 'assgn',\n",
       " 'assgnmt',\n",
       " 'assi',\n",
       " 'assign',\n",
       " 'assignment',\n",
       " 'assignments',\n",
       " 'assis',\n",
       " 'assist',\n",
       " 'assista',\n",
       " 'assistan',\n",
       " 'assistance',\n",
       " 'assistane',\n",
       " 'assistant',\n",
       " 'assistants',\n",
       " 'assistive',\n",
       " 'assitant',\n",
       " 'assmt',\n",
       " 'assn',\n",
       " 'assoc',\n",
       " 'associate',\n",
       " 'associated',\n",
       " 'asst',\n",
       " 'assur',\n",
       " 'assurance',\n",
       " 'asthma',\n",
       " 'at',\n",
       " 'ath',\n",
       " 'athl',\n",
       " 'athletic',\n",
       " 'athletics',\n",
       " 'atrt',\n",
       " 'atsp',\n",
       " 'attendance',\n",
       " 'attendant',\n",
       " 'attendence',\n",
       " 'attn',\n",
       " 'attorney',\n",
       " 'attorneys',\n",
       " 'atwll',\n",
       " 'audio',\n",
       " 'audiologi',\n",
       " 'audiologist',\n",
       " 'audiology',\n",
       " 'audiovisual',\n",
       " 'audit',\n",
       " 'auditing',\n",
       " 'auditor',\n",
       " 'auditory',\n",
       " 'aug',\n",
       " 'august',\n",
       " 'autism',\n",
       " 'autistic',\n",
       " 'auto',\n",
       " 'autobody',\n",
       " 'automation',\n",
       " 'automechanics',\n",
       " 'automotive',\n",
       " 'autonamy',\n",
       " 'autonomous',\n",
       " 'aux',\n",
       " 'auxiliary',\n",
       " 'av',\n",
       " 'avenue',\n",
       " 'avg',\n",
       " 'aviation',\n",
       " 'avid',\n",
       " 'award',\n",
       " 'awards',\n",
       " 'ayp',\n",
       " 'b',\n",
       " 'ba',\n",
       " 'baccalaur',\n",
       " 'baccalaurate',\n",
       " 'bachelor',\n",
       " 'back',\n",
       " 'background',\n",
       " 'backpack',\n",
       " 'baker',\n",
       " 'bakery',\n",
       " 'band',\n",
       " 'bank',\n",
       " 'banking',\n",
       " 'banquets',\n",
       " 'barbering',\n",
       " 'barrier',\n",
       " 'baseball',\n",
       " 'based',\n",
       " 'basic',\n",
       " 'basis',\n",
       " 'basketball',\n",
       " 'bbb',\n",
       " 'bcbs',\n",
       " 'bckflw',\n",
       " 'bd',\n",
       " 'bdgt',\n",
       " 'be',\n",
       " 'beat',\n",
       " 'beginning',\n",
       " 'behalf',\n",
       " 'behav',\n",
       " 'behavior',\n",
       " 'behavioral',\n",
       " 'behavorial',\n",
       " 'believers',\n",
       " 'bell',\n",
       " 'benchmark',\n",
       " 'benef',\n",
       " 'benefit',\n",
       " 'benefits',\n",
       " 'best',\n",
       " 'beyond',\n",
       " 'bga',\n",
       " 'bgt',\n",
       " 'bi',\n",
       " 'bible',\n",
       " 'bicycle',\n",
       " 'bil',\n",
       " 'biling',\n",
       " 'bilingua',\n",
       " 'bilingual',\n",
       " 'bilinguial',\n",
       " 'billing',\n",
       " 'bilng',\n",
       " 'bilngl',\n",
       " 'binding',\n",
       " 'binligual',\n",
       " 'biological',\n",
       " 'biology',\n",
       " 'bird',\n",
       " 'birth',\n",
       " 'bks',\n",
       " 'black',\n",
       " 'bldg',\n",
       " 'bldgs',\n",
       " 'blind',\n",
       " 'blk',\n",
       " 'blngl',\n",
       " 'block',\n",
       " 'bo',\n",
       " 'board',\n",
       " 'body',\n",
       " 'boiler',\n",
       " 'boilermaker',\n",
       " 'boilers',\n",
       " 'bolton',\n",
       " 'bond',\n",
       " 'bonding',\n",
       " 'bonds',\n",
       " 'bonus',\n",
       " 'bonuses',\n",
       " 'book',\n",
       " 'bookkeeper',\n",
       " 'bookkeepers',\n",
       " 'books',\n",
       " 'boost',\n",
       " 'boot',\n",
       " 'bottled',\n",
       " 'boy',\n",
       " 'boys',\n",
       " 'bp',\n",
       " 'braille',\n",
       " 'brd',\n",
       " 'breaker',\n",
       " 'breakfast',\n",
       " 'brick',\n",
       " 'bricklayer',\n",
       " 'bricks',\n",
       " 'bridge',\n",
       " 'bright',\n",
       " 'brkfast',\n",
       " 'broad',\n",
       " 'broadcast',\n",
       " 'broadcasting',\n",
       " 'broken',\n",
       " 'brothers',\n",
       " 'bs',\n",
       " 'bsi',\n",
       " 'budg',\n",
       " 'budget',\n",
       " 'budgetary',\n",
       " 'budgeting',\n",
       " 'budgets',\n",
       " 'buhrer',\n",
       " 'building',\n",
       " 'buildings',\n",
       " 'buisness',\n",
       " 'bulk',\n",
       " 'bump',\n",
       " 'bunch',\n",
       " 'bureau',\n",
       " 'bus',\n",
       " 'buses',\n",
       " 'busine',\n",
       " 'business',\n",
       " 'buy',\n",
       " 'buyer',\n",
       " 'by',\n",
       " 'c',\n",
       " 'cabinet',\n",
       " 'cabinetmaker',\n",
       " 'cable',\n",
       " 'cad',\n",
       " 'cadd',\n",
       " 'cadre',\n",
       " 'cafe',\n",
       " 'cafeteria',\n",
       " 'camera',\n",
       " 'camp',\n",
       " 'campbell',\n",
       " 'camps',\n",
       " 'campus',\n",
       " 'candies',\n",
       " 'candle',\n",
       " 'candy',\n",
       " 'cao',\n",
       " 'cap',\n",
       " 'capacity',\n",
       " 'capital',\n",
       " 'capitalized',\n",
       " 'captain',\n",
       " 'card',\n",
       " 'cards',\n",
       " 'care',\n",
       " 'careeer',\n",
       " 'career',\n",
       " 'caregiver',\n",
       " 'caretaker',\n",
       " 'carl',\n",
       " 'carnegie',\n",
       " 'carpenter',\n",
       " 'carpentry',\n",
       " 'carpet',\n",
       " 'carreer',\n",
       " 'carrier',\n",
       " 'carryforward',\n",
       " 'carryover',\n",
       " 'case',\n",
       " 'cash',\n",
       " 'cashier',\n",
       " 'catalyst',\n",
       " 'category',\n",
       " 'catholic',\n",
       " 'cclc',\n",
       " 'ccoach',\n",
       " 'cdb',\n",
       " 'cdc',\n",
       " 'ce',\n",
       " 'cec',\n",
       " 'ceis',\n",
       " 'cela',\n",
       " 'celeb',\n",
       " 'cen',\n",
       " 'cent',\n",
       " 'center',\n",
       " 'centers',\n",
       " 'central',\n",
       " 'century',\n",
       " 'ceo',\n",
       " 'ceramic',\n",
       " 'cert',\n",
       " 'certificated',\n",
       " 'certification',\n",
       " 'certified',\n",
       " 'certifiedaddl',\n",
       " 'cf',\n",
       " 'cfda',\n",
       " 'cfo',\n",
       " 'ch',\n",
       " 'cha',\n",
       " 'chai',\n",
       " 'chair',\n",
       " 'challenge',\n",
       " 'champions',\n",
       " 'chance',\n",
       " 'chancellor',\n",
       " 'chanclr',\n",
       " 'change',\n",
       " 'chapter',\n",
       " 'char',\n",
       " 'character',\n",
       " 'charge',\n",
       " 'charges',\n",
       " 'charitable',\n",
       " 'charities',\n",
       " 'charity',\n",
       " 'charter',\n",
       " 'charters',\n",
       " 'charting',\n",
       " 'chdrn',\n",
       " 'che',\n",
       " 'cheerlea',\n",
       " 'cheerleader',\n",
       " 'cheerleaders',\n",
       " 'cheerleading',\n",
       " 'chef',\n",
       " 'chem',\n",
       " 'chemical',\n",
       " 'chemistry',\n",
       " 'chief',\n",
       " 'child',\n",
       " 'childcare',\n",
       " 'childhd',\n",
       " 'childhood',\n",
       " 'children',\n",
       " 'chinese',\n",
       " 'chllnge',\n",
       " 'choice',\n",
       " 'choir',\n",
       " 'choral',\n",
       " 'chorus',\n",
       " 'chosen',\n",
       " 'christian',\n",
       " 'chrtr',\n",
       " 'church',\n",
       " 'cie',\n",
       " 'cisco',\n",
       " 'city',\n",
       " 'civics',\n",
       " 'claiming',\n",
       " 'claims',\n",
       " 'class',\n",
       " 'classified',\n",
       " 'classroom',\n",
       " 'classroon',\n",
       " 'clean',\n",
       " 'cleaner',\n",
       " 'cleaners',\n",
       " 'cleaning',\n",
       " 'clerical',\n",
       " 'clerk',\n",
       " 'client',\n",
       " 'climate',\n",
       " 'climbing',\n",
       " 'clinic',\n",
       " 'clinical',\n",
       " 'clk',\n",
       " 'clos',\n",
       " 'closed',\n",
       " 'closeures',\n",
       " 'closing',\n",
       " 'clothing',\n",
       " 'clout',\n",
       " 'cls',\n",
       " 'club',\n",
       " 'cluster',\n",
       " 'cms',\n",
       " 'cmsd',\n",
       " 'cnflct',\n",
       " 'cnfrn',\n",
       " 'cnslg',\n",
       " 'cntr',\n",
       " 'cntrl',\n",
       " 'cntry',\n",
       " 'co',\n",
       " 'coa',\n",
       " 'coac',\n",
       " 'coach',\n",
       " 'coaches',\n",
       " 'coaching',\n",
       " 'coalit',\n",
       " 'coalition',\n",
       " 'coappp',\n",
       " 'coatings',\n",
       " 'cocurricular',\n",
       " 'code',\n",
       " 'cohart',\n",
       " 'cohort',\n",
       " 'coll',\n",
       " 'collabo',\n",
       " 'collabor',\n",
       " 'collaboratives',\n",
       " 'collection',\n",
       " 'collections',\n",
       " 'college',\n",
       " 'collegiate',\n",
       " 'colleyball',\n",
       " 'collision',\n",
       " 'colorado',\n",
       " 'com',\n",
       " 'combination',\n",
       " 'combustion',\n",
       " 'comes',\n",
       " 'comm',\n",
       " 'command',\n",
       " 'commerc',\n",
       " 'commission',\n",
       " 'commissions',\n",
       " 'commodities',\n",
       " 'commodity',\n",
       " 'common',\n",
       " 'commty',\n",
       " 'communicatio',\n",
       " 'communication',\n",
       " 'communications',\n",
       " 'communities',\n",
       " 'community',\n",
       " 'comp',\n",
       " 'compass',\n",
       " 'compen',\n",
       " 'compensation',\n",
       " 'compensatory',\n",
       " 'competency',\n",
       " 'competition',\n",
       " 'competitive',\n",
       " 'compl',\n",
       " 'completion',\n",
       " 'complex',\n",
       " 'compliance',\n",
       " 'composition',\n",
       " 'comprehensive',\n",
       " 'comptr',\n",
       " 'comptrolle',\n",
       " 'compute',\n",
       " 'computer',\n",
       " 'computers',\n",
       " 'computing',\n",
       " 'concepts',\n",
       " 'concession',\n",
       " 'concessions',\n",
       " 'concussion',\n",
       " 'condiments',\n",
       " 'condition',\n",
       " 'conf',\n",
       " 'confer',\n",
       " 'conference',\n",
       " 'conflict',\n",
       " 'cong',\n",
       " 'connect',\n",
       " 'connections',\n",
       " 'conservation',\n",
       " 'conslt',\n",
       " 'consolidated',\n",
       " 'consolidation',\n",
       " 'consolidations',\n",
       " 'consortium',\n",
       " 'const',\n",
       " 'constituency',\n",
       " 'constr',\n",
       " 'constrct',\n",
       " 'construction',\n",
       " 'constuction',\n",
       " 'consul',\n",
       " 'consultant',\n",
       " 'consultants',\n",
       " 'consumer',\n",
       " 'cont',\n",
       " 'contact',\n",
       " 'containd',\n",
       " 'contained',\n",
       " 'contemporary',\n",
       " 'content',\n",
       " 'contingencies',\n",
       " 'contingency',\n",
       " 'continuing',\n",
       " 'contr',\n",
       " 'contra',\n",
       " 'contract',\n",
       " 'contracted',\n",
       " 'contracting',\n",
       " 'contractor',\n",
       " 'contractors',\n",
       " 'contracts',\n",
       " 'contractual',\n",
       " 'contributing',\n",
       " 'contribution',\n",
       " 'contro',\n",
       " 'control',\n",
       " 'controller',\n",
       " 'controls',\n",
       " 'conversion',\n",
       " 'coo',\n",
       " 'cook',\n",
       " 'cooperating',\n",
       " 'coor',\n",
       " 'coord',\n",
       " 'coordinated',\n",
       " 'coordination',\n",
       " 'coordinator',\n",
       " 'cop',\n",
       " 'copayment',\n",
       " 'copier',\n",
       " 'copiers',\n",
       " 'cops',\n",
       " 'copy',\n",
       " 'copying',\n",
       " 'core',\n",
       " 'cornerstone',\n",
       " 'corp',\n",
       " 'corporal',\n",
       " 'corps',\n",
       " 'correction',\n",
       " 'correctionist',\n",
       " 'cos',\n",
       " 'cosmetology',\n",
       " 'cosmotology',\n",
       " 'cost',\n",
       " 'costs',\n",
       " 'costume',\n",
       " 'coteaching',\n",
       " 'council',\n",
       " 'couns',\n",
       " 'counsel',\n",
       " 'counseling',\n",
       " 'counselor',\n",
       " 'counselors',\n",
       " 'count',\n",
       " 'counter',\n",
       " 'counterman',\n",
       " 'countr',\n",
       " 'country',\n",
       " 'county',\n",
       " 'coupon',\n",
       " 'court',\n",
       " 'coverage',\n",
       " 'covered',\n",
       " 'cpa',\n",
       " 'craft',\n",
       " 'crdt',\n",
       " 'create',\n",
       " 'creating',\n",
       " 'creative',\n",
       " 'credit',\n",
       " 'crew',\n",
       " 'criminal',\n",
       " 'crisis',\n",
       " 'crit',\n",
       " 'critical',\n",
       " 'cross',\n",
       " 'crossing',\n",
       " 'crt',\n",
       " 'crystal',\n",
       " 'cst',\n",
       " 'csx',\n",
       " 'ctae',\n",
       " 'cte',\n",
       " 'cti',\n",
       " 'ctl',\n",
       " 'cto',\n",
       " 'ctr',\n",
       " 'ctu',\n",
       " 'cty',\n",
       " 'culinary',\n",
       " 'cult',\n",
       " 'culture',\n",
       " 'cultures',\n",
       " 'curator',\n",
       " 'curr',\n",
       " 'current',\n",
       " 'curricular',\n",
       " 'curriculum',\n",
       " 'cus',\n",
       " 'cusotidan',\n",
       " 'cust',\n",
       " 'custodain',\n",
       " 'custodial',\n",
       " 'custodian',\n",
       " 'custodians',\n",
       " 'customer',\n",
       " 'customized',\n",
       " 'cutters',\n",
       " 'd',\n",
       " 'da',\n",
       " 'daep',\n",
       " 'daily',\n",
       " 'damaged',\n",
       " 'dance',\n",
       " 'data',\n",
       " 'database',\n",
       " 'datacoord',\n",
       " 'day',\n",
       " 'days',\n",
       " 'dc',\n",
       " 'dcta',\n",
       " 'deaf',\n",
       " 'dean',\n",
       " 'debate',\n",
       " 'debt',\n",
       " 'dec',\n",
       " 'decathlon',\n",
       " 'decis',\n",
       " 'decision',\n",
       " 'decrease',\n",
       " 'dedicated',\n",
       " 'deductible',\n",
       " 'deep',\n",
       " 'def',\n",
       " 'defined',\n",
       " 'degreed',\n",
       " 'del',\n",
       " 'deli',\n",
       " 'delinq',\n",
       " 'delinquent',\n",
       " 'delivery',\n",
       " 'dell',\n",
       " 'delta',\n",
       " 'demnd',\n",
       " 'demolition',\n",
       " 'dental',\n",
       " 'dep',\n",
       " 'department',\n",
       " 'departments',\n",
       " 'depot',\n",
       " 'depreciation',\n",
       " 'dept',\n",
       " 'depty',\n",
       " 'deputy',\n",
       " 'des',\n",
       " 'description',\n",
       " 'desgn',\n",
       " 'design',\n",
       " 'designated',\n",
       " 'desk',\n",
       " 'desktop',\n",
       " 'destination',\n",
       " 'destructive',\n",
       " 'detective',\n",
       " 'determ',\n",
       " 'determination',\n",
       " 'dev',\n",
       " 'develop',\n",
       " 'develope',\n",
       " 'developer',\n",
       " 'development',\n",
       " 'developmentally',\n",
       " 'develpmnt',\n",
       " 'develpmt',\n",
       " 'devices',\n",
       " 'devp',\n",
       " 'dhh',\n",
       " 'dhr',\n",
       " 'di',\n",
       " 'diabetes',\n",
       " 'diagnostic',\n",
       " 'diagnostician',\n",
       " 'diagnosticians',\n",
       " 'diags',\n",
       " 'diesel',\n",
       " 'dietary',\n",
       " 'dietician',\n",
       " 'diff',\n",
       " 'differential',\n",
       " 'differentiated',\n",
       " 'digital',\n",
       " 'dinner',\n",
       " 'dipl',\n",
       " 'diploma',\n",
       " 'diplomas',\n",
       " 'dir',\n",
       " 'direc',\n",
       " 'direct',\n",
       " 'direction',\n",
       " 'directo',\n",
       " 'director',\n",
       " 'directory',\n",
       " 'directr',\n",
       " 'dis',\n",
       " 'disabilit',\n",
       " 'disabilities',\n",
       " 'disability',\n",
       " 'disabled',\n",
       " 'disadvantaged',\n",
       " 'disbursing',\n",
       " 'discharged',\n",
       " 'disciplinary',\n",
       " 'discipline',\n",
       " 'discovery',\n",
       " 'discretionary',\n",
       " 'dislocated',\n",
       " 'disord',\n",
       " 'disorder',\n",
       " 'disorders',\n",
       " 'dispatch',\n",
       " 'dispatcher',\n",
       " 'dispensing',\n",
       " 'disposal',\n",
       " 'dist',\n",
       " 'distance',\n",
       " 'distinction',\n",
       " 'distinguished',\n",
       " 'distributed',\n",
       " 'distribution',\n",
       " 'district',\n",
       " 'districts',\n",
       " 'districtwide',\n",
       " 'div',\n",
       " 'diversified',\n",
       " 'diversity',\n",
       " 'division',\n",
       " 'do',\n",
       " 'doctoral',\n",
       " 'doctorate',\n",
       " 'document',\n",
       " 'doe',\n",
       " 'dollar',\n",
       " 'dollars',\n",
       " 'domestic',\n",
       " 'donat',\n",
       " 'donated',\n",
       " 'donation',\n",
       " 'donations',\n",
       " 'donors',\n",
       " 'dop',\n",
       " 'dot',\n",
       " 'dots',\n",
       " 'dr',\n",
       " 'drama',\n",
       " 'dream',\n",
       " 'driver',\n",
       " 'drivers',\n",
       " 'drnge',\n",
       " 'drop',\n",
       " 'dropout',\n",
       " 'drug',\n",
       " 'drum',\n",
       " 'ds',\n",
       " 'dual',\n",
       " 'duck',\n",
       " 'dues',\n",
       " 'duplicating',\n",
       " 'duplicator',\n",
       " 'duty',\n",
       " 'dvlp',\n",
       " 'dvlpmnt',\n",
       " 'dyslexia',\n",
       " 'e',\n",
       " 'each',\n",
       " 'early',\n",
       " 'earth',\n",
       " 'east',\n",
       " 'eaton',\n",
       " 'ebd',\n",
       " 'ec',\n",
       " 'ece',\n",
       " 'ecea',\n",
       " 'econ',\n",
       " 'economic',\n",
       " 'economics',\n",
       " 'ecse',\n",
       " 'ed',\n",
       " 'edcuational',\n",
       " 'editor',\n",
       " 'edu',\n",
       " 'educ',\n",
       " 'educat',\n",
       " 'education',\n",
       " 'educational',\n",
       " 'educatoinal',\n",
       " 'educator',\n",
       " 'educators',\n",
       " 'eett',\n",
       " 'eff',\n",
       " 'effective',\n",
       " 'effectiveness',\n",
       " 'eg',\n",
       " 'eii',\n",
       " 'eis',\n",
       " 'el',\n",
       " 'ela',\n",
       " 'elc',\n",
       " 'election',\n",
       " 'electr',\n",
       " 'electrical',\n",
       " 'electrician',\n",
       " ...]"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vec_alphanumeric.get_feature_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Function                                       NO_LABEL\n",
       "Use                                            NO_LABEL\n",
       "Sharing                                        NO_LABEL\n",
       "Reporting                                      NO_LABEL\n",
       "Student_Type                                   NO_LABEL\n",
       "Position_Type                                  NO_LABEL\n",
       "Object_Type                                    NO_LABEL\n",
       "Pre_K                                          NO_LABEL\n",
       "Operating_Status                          Non-Operating\n",
       "Object_Description                  CONTRACTOR SERVICES\n",
       "Text_2                                BOND EXPENDITURES\n",
       "SubFund_Description                       BUILDING FUND\n",
       "Job_Title_Description                           (blank)\n",
       "Text_3                                          Regular\n",
       "Text_4                                              NaN\n",
       "Sub_Object_Description                              NaN\n",
       "Location_Description                                NaN\n",
       "FTE                                                 NaN\n",
       "Function_Description                           RGN  GOB\n",
       "Facility_or_Department                              NaN\n",
       "Position_Extra                             UNDESIGNATED\n",
       "Total                                           3477.86\n",
       "Program_Description       BUILDING IMPROVEMENT SERVICES\n",
       "Fund_Description                                    NaN\n",
       "Text_1                    BUILDING IMPROVEMENT SERVICES\n",
       "Name: 206341, dtype: object"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.iloc[1, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4758"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(vec_basic.get_feature_names())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3284"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(vec_alphanumeric.get_feature_names())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Tokenizing on alpha-numeric tokens reduces the number of token. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Let's build a pipeline\n",
    "\n",
    "The Pipeline workflow is a \n",
    "\n",
    "* Repeatable way to go from raw data to trained model\n",
    "\n",
    "* Pipieline object takes sequential list of steps\n",
    "\n",
    "* Output of one step is input to next step\n",
    "\n",
    "* Each step is a tuple with two elements -- the name which is a string, and an object that implements .fit() and .transform() methods\n",
    "\n",
    "* Pipeline are a flexible way to represent the workflow. In fact, a step can itself be another pipeline. \n",
    "\n",
    "The problem here is that the text and numeric data don't work together. For instance, CountVectorizer doesn't know what to do with numeric data and Imputer doesn't know how to imput on text columns. \n",
    "\n",
    "Solution: buid two different pipelines using FunctionTransformer and put them together using FeatureUnion. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['FTE',\n",
       " 'Total',\n",
       " 'Function',\n",
       " 'Use',\n",
       " 'Sharing',\n",
       " 'Reporting',\n",
       " 'Student_Type',\n",
       " 'Position_Type',\n",
       " 'Object_Type',\n",
       " 'Pre_K',\n",
       " 'Operating_Status']"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "NUMERIC_COLUMNS + LABELS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ALL_COLUMNS = df_train.columns.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "TEXT_COLUMNS = list(np.setdiff1d(ALL_COLUMNS, NUMERIC_COLUMNS + LABELS))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Facility_or_Department',\n",
       " 'Function_Description',\n",
       " 'Fund_Description',\n",
       " 'Job_Title_Description',\n",
       " 'Location_Description',\n",
       " 'Object_Description',\n",
       " 'Position_Extra',\n",
       " 'Program_Description',\n",
       " 'SubFund_Description',\n",
       " 'Sub_Object_Description',\n",
       " 'Text_1',\n",
       " 'Text_2',\n",
       " 'Text_3',\n",
       " 'Text_4']"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TEXT_COLUMNS"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note we need to use the combine_text_columns function because our tokenizer accepts one string."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "get_text_data = FunctionTransformer(combine_text_columns, validate = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "FunctionTransformer job is to take a Python function and turn it into an object that a scikit-learn pipeline can understand.\n",
    "\n",
    "The validate=False tells scikit-learn it doesn't need to check for NaNs or validate the dtypes of the input. We will do that work outselves."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "get_numeric_data = FunctionTransformer(lambda x: x[NUMERIC_COLUMNS], validate = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "FunctionTransformer(accept_sparse=False,\n",
       "          func=<function <lambda> at 0x12fb00158>, inv_kw_args=None,\n",
       "          inverse_func=None, kw_args=None, pass_y='deprecated',\n",
       "          validate=False)"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_numeric_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Facility_or_Department</th>\n",
       "      <th>Function_Description</th>\n",
       "      <th>Fund_Description</th>\n",
       "      <th>Job_Title_Description</th>\n",
       "      <th>Location_Description</th>\n",
       "      <th>Object_Description</th>\n",
       "      <th>Position_Extra</th>\n",
       "      <th>Program_Description</th>\n",
       "      <th>SubFund_Description</th>\n",
       "      <th>Sub_Object_Description</th>\n",
       "      <th>Text_1</th>\n",
       "      <th>Text_2</th>\n",
       "      <th>Text_3</th>\n",
       "      <th>Text_4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>134338</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>General Fund</td>\n",
       "      <td>Teacher-Elementary</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>KINDERGARTEN</td>\n",
       "      <td>KINDERGARTEN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>206341</th>\n",
       "      <td>NaN</td>\n",
       "      <td>RGN  GOB</td>\n",
       "      <td>NaN</td>\n",
       "      <td>(blank)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>CONTRACTOR SERVICES</td>\n",
       "      <td>UNDESIGNATED</td>\n",
       "      <td>BUILDING IMPROVEMENT SERVICES</td>\n",
       "      <td>BUILDING FUND</td>\n",
       "      <td>NaN</td>\n",
       "      <td>BUILDING IMPROVEMENT SERVICES</td>\n",
       "      <td>BOND EXPENDITURES</td>\n",
       "      <td>Regular</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>326408</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>General Purpose School</td>\n",
       "      <td>TCHER 2ND GRADE</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Personal Services - Teachers</td>\n",
       "      <td>TEACHER</td>\n",
       "      <td>Instruction - Regular</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Regular Instruction</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>364634</th>\n",
       "      <td>NaN</td>\n",
       "      <td>UNALLOC BUDGETS/SCHOOLS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Teacher, Short Term Sub</td>\n",
       "      <td>NaN</td>\n",
       "      <td>EMPLOYEE BENEFITS</td>\n",
       "      <td>PROFESSIONAL-INSTRUCTIONAL</td>\n",
       "      <td>GENERAL MIDDLE/JUNIOR HIGH SCH</td>\n",
       "      <td>GENERAL FUND</td>\n",
       "      <td>NaN</td>\n",
       "      <td>REGULAR INSTRUCTION</td>\n",
       "      <td>TEACHER SUBS</td>\n",
       "      <td>Regular</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47683</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NON-PROJECT</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Teacher, Secondary (High)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>TEACHER COVERAGE FOR TEACHER</td>\n",
       "      <td>PROFESSIONAL-INSTRUCTIONAL</td>\n",
       "      <td>GENERAL HIGH SCHOOL EDUCATION</td>\n",
       "      <td>GENERAL FUND</td>\n",
       "      <td>NaN</td>\n",
       "      <td>REGULAR INSTRUCTION</td>\n",
       "      <td>TEACHER SUBS</td>\n",
       "      <td>Alternative</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Facility_or_Department     Function_Description  \\\n",
       "134338                    NaN                      NaN   \n",
       "206341                    NaN                 RGN  GOB   \n",
       "326408                    NaN                      NaN   \n",
       "364634                    NaN  UNALLOC BUDGETS/SCHOOLS   \n",
       "47683                     NaN              NON-PROJECT   \n",
       "\n",
       "              Fund_Description       Job_Title_Description  \\\n",
       "134338            General Fund         Teacher-Elementary    \n",
       "206341                     NaN                     (blank)   \n",
       "326408  General Purpose School             TCHER 2ND GRADE   \n",
       "364634                     NaN    Teacher, Short Term Sub    \n",
       "47683                      NaN  Teacher, Secondary (High)    \n",
       "\n",
       "       Location_Description            Object_Description  \\\n",
       "134338                  NaN                           NaN   \n",
       "206341                  NaN           CONTRACTOR SERVICES   \n",
       "326408                  NaN  Personal Services - Teachers   \n",
       "364634                  NaN             EMPLOYEE BENEFITS   \n",
       "47683                   NaN  TEACHER COVERAGE FOR TEACHER   \n",
       "\n",
       "                    Position_Extra             Program_Description  \\\n",
       "134338               KINDERGARTEN                     KINDERGARTEN   \n",
       "206341                UNDESIGNATED   BUILDING IMPROVEMENT SERVICES   \n",
       "326408                     TEACHER           Instruction - Regular   \n",
       "364634  PROFESSIONAL-INSTRUCTIONAL  GENERAL MIDDLE/JUNIOR HIGH SCH   \n",
       "47683   PROFESSIONAL-INSTRUCTIONAL   GENERAL HIGH SCHOOL EDUCATION   \n",
       "\n",
       "       SubFund_Description Sub_Object_Description  \\\n",
       "134338                 NaN                    NaN   \n",
       "206341       BUILDING FUND                    NaN   \n",
       "326408                 NaN                    NaN   \n",
       "364634        GENERAL FUND                    NaN   \n",
       "47683         GENERAL FUND                    NaN   \n",
       "\n",
       "                               Text_1             Text_2       Text_3  \\\n",
       "134338                            NaN                NaN          NaN   \n",
       "206341  BUILDING IMPROVEMENT SERVICES  BOND EXPENDITURES      Regular   \n",
       "326408                            NaN                NaN          NaN   \n",
       "364634            REGULAR INSTRUCTION       TEACHER SUBS      Regular   \n",
       "47683             REGULAR INSTRUCTION       TEACHER SUBS  Alternative   \n",
       "\n",
       "                     Text_4  \n",
       "134338                  NaN  \n",
       "206341                  NaN  \n",
       "326408  Regular Instruction  \n",
       "364634                  NaN  \n",
       "47683                   NaN  "
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train[TEXT_COLUMNS].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "just_text_data = get_text_data.fit_transform(df_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "134338       Teacher-Elementary        KINDERGARTEN  KIN...\n",
       "206341    CONTRACTOR SERVICES BOND EXPENDITURES BUILDING...\n",
       "326408    Personal Services - Teachers   TCHER 2ND GRADE...\n",
       "364634    EMPLOYEE BENEFITS TEACHER SUBS GENERAL FUND Te...\n",
       "47683     TEACHER COVERAGE FOR TEACHER TEACHER SUBS GENE...\n",
       "dtype: object"
      ]
     },
     "execution_count": 139,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "just_text_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(400277,)"
      ]
     },
     "execution_count": 140,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "just_text_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(TEXT_COLUMNS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "just_numeric_data = get_numeric_data.fit_transform(df_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(400277, 2)"
      ]
     },
     "execution_count": 143,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "just_numeric_data.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Using FeatureUnion()\n",
    "\n",
    "We combine the multiple type of feature processing using the FeatureUnion utility for scikit-learn.\n",
    "\n",
    "Before that we use the multilabel_train_test_split() we defined above on all of the data set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = multilabel_train_test_split(df_train[NUMERIC_COLUMNS + TEXT_COLUMNS],\n",
    "                                                               pd.get_dummies(df_train[LABELS]),\n",
    "                                                               size=0.2, \n",
    "                                                               seed=123)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** Create a FeatureUnion with nested pipeline **"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "process_and_join_features = FeatureUnion(\n",
    "                            transformer_list = [\n",
    "                                ('numeric_features', Pipeline([\n",
    "                                    ('selector', get_numeric_data),\n",
    "                                    ('imputer', Imputer())\n",
    "                                ])), \n",
    "                                ('text_features', Pipeline([\n",
    "                                    ('selector', get_text_data),\n",
    "                                    ('vectorizer', CountVectorizer())\n",
    "                                ]))\n",
    "                            ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "process_and_join_features_alphanum = FeatureUnion(\n",
    "                            transformer_list = [\n",
    "                                ('numeric_features', Pipeline([\n",
    "                                    ('selector', get_numeric_data),\n",
    "                                    ('imputer', Imputer())\n",
    "                                ])), \n",
    "                                ('text_features', Pipeline([\n",
    "                                    ('selector', get_text_data),\n",
    "                                    ('vectorizer', CountVectorizer(token_pattern=TOKENS_ALPHANUMERIC, ngram_range=(1, 2)))\n",
    "                                ]))\n",
    "                            ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "process_and_join_features_basic = FeatureUnion(\n",
    "                            transformer_list = [\n",
    "                                ('numeric_features', Pipeline([\n",
    "                                    ('selector', get_numeric_data),\n",
    "                                    ('imputer', Imputer())\n",
    "                                ])), \n",
    "                                ('text_features', Pipeline([\n",
    "                                    ('selector', get_text_data),\n",
    "                                    ('vectorizer', CountVectorizer(token_pattern=TOKENS_BASIC))\n",
    "                                ]))\n",
    "                            ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pl = Pipeline([\n",
    "    ('union', process_and_join_features),\n",
    "    ('clf', OneVsRestClassifier(LogisticRegression()))\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pl_alphanum = Pipeline([\n",
    "    ('union', process_and_join_features_alphanum),\n",
    "    ('clf', OneVsRestClassifier(LogisticRegression()))\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pl_basic = Pipeline([\n",
    "    ('union', process_and_join_features_basic),\n",
    "    ('clf', OneVsRestClassifier(LogisticRegression()))\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>FTE</th>\n",
       "      <th>Total</th>\n",
       "      <th>Facility_or_Department</th>\n",
       "      <th>Function_Description</th>\n",
       "      <th>Fund_Description</th>\n",
       "      <th>Job_Title_Description</th>\n",
       "      <th>Location_Description</th>\n",
       "      <th>Object_Description</th>\n",
       "      <th>Position_Extra</th>\n",
       "      <th>Program_Description</th>\n",
       "      <th>SubFund_Description</th>\n",
       "      <th>Sub_Object_Description</th>\n",
       "      <th>Text_1</th>\n",
       "      <th>Text_2</th>\n",
       "      <th>Text_3</th>\n",
       "      <th>Text_4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>134338</th>\n",
       "      <td>1.0</td>\n",
       "      <td>50471.810</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>General Fund</td>\n",
       "      <td>Teacher-Elementary</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>KINDERGARTEN</td>\n",
       "      <td>KINDERGARTEN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>326408</th>\n",
       "      <td>1.0</td>\n",
       "      <td>62237.130</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>General Purpose School</td>\n",
       "      <td>TCHER 2ND GRADE</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Personal Services - Teachers</td>\n",
       "      <td>TEACHER</td>\n",
       "      <td>Instruction - Regular</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Regular Instruction</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>364634</th>\n",
       "      <td>NaN</td>\n",
       "      <td>22.300</td>\n",
       "      <td>NaN</td>\n",
       "      <td>UNALLOC BUDGETS/SCHOOLS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Teacher, Short Term Sub</td>\n",
       "      <td>NaN</td>\n",
       "      <td>EMPLOYEE BENEFITS</td>\n",
       "      <td>PROFESSIONAL-INSTRUCTIONAL</td>\n",
       "      <td>GENERAL MIDDLE/JUNIOR HIGH SCH</td>\n",
       "      <td>GENERAL FUND</td>\n",
       "      <td>NaN</td>\n",
       "      <td>REGULAR INSTRUCTION</td>\n",
       "      <td>TEACHER SUBS</td>\n",
       "      <td>Regular</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47683</th>\n",
       "      <td>NaN</td>\n",
       "      <td>54.166</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NON-PROJECT</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Teacher, Secondary (High)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>TEACHER COVERAGE FOR TEACHER</td>\n",
       "      <td>PROFESSIONAL-INSTRUCTIONAL</td>\n",
       "      <td>GENERAL HIGH SCHOOL EDUCATION</td>\n",
       "      <td>GENERAL FUND</td>\n",
       "      <td>NaN</td>\n",
       "      <td>REGULAR INSTRUCTION</td>\n",
       "      <td>TEACHER SUBS</td>\n",
       "      <td>Alternative</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>229958</th>\n",
       "      <td>NaN</td>\n",
       "      <td>-8.150</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NON-PROJECT</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Custodian - PT -  Jobs</td>\n",
       "      <td>NaN</td>\n",
       "      <td>CONTRA BENEFITS</td>\n",
       "      <td>UNDESIGNATED</td>\n",
       "      <td>EMPLOYEE BENEFITS</td>\n",
       "      <td>GENERAL FUND</td>\n",
       "      <td>NaN</td>\n",
       "      <td>EMPLOYEE BENEFITS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>n/a</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        FTE      Total Facility_or_Department     Function_Description  \\\n",
       "134338  1.0  50471.810                    NaN                      NaN   \n",
       "326408  1.0  62237.130                    NaN                      NaN   \n",
       "364634  NaN     22.300                    NaN  UNALLOC BUDGETS/SCHOOLS   \n",
       "47683   NaN     54.166                    NaN              NON-PROJECT   \n",
       "229958  NaN     -8.150                    NaN              NON-PROJECT   \n",
       "\n",
       "              Fund_Description       Job_Title_Description  \\\n",
       "134338            General Fund         Teacher-Elementary    \n",
       "326408  General Purpose School             TCHER 2ND GRADE   \n",
       "364634                     NaN    Teacher, Short Term Sub    \n",
       "47683                      NaN  Teacher, Secondary (High)    \n",
       "229958                     NaN     Custodian - PT -  Jobs    \n",
       "\n",
       "       Location_Description            Object_Description  \\\n",
       "134338                  NaN                           NaN   \n",
       "326408                  NaN  Personal Services - Teachers   \n",
       "364634                  NaN             EMPLOYEE BENEFITS   \n",
       "47683                   NaN  TEACHER COVERAGE FOR TEACHER   \n",
       "229958                  NaN               CONTRA BENEFITS   \n",
       "\n",
       "                    Position_Extra             Program_Description  \\\n",
       "134338               KINDERGARTEN                     KINDERGARTEN   \n",
       "326408                     TEACHER           Instruction - Regular   \n",
       "364634  PROFESSIONAL-INSTRUCTIONAL  GENERAL MIDDLE/JUNIOR HIGH SCH   \n",
       "47683   PROFESSIONAL-INSTRUCTIONAL   GENERAL HIGH SCHOOL EDUCATION   \n",
       "229958                UNDESIGNATED               EMPLOYEE BENEFITS   \n",
       "\n",
       "       SubFund_Description Sub_Object_Description               Text_1  \\\n",
       "134338                 NaN                    NaN                  NaN   \n",
       "326408                 NaN                    NaN                  NaN   \n",
       "364634        GENERAL FUND                    NaN  REGULAR INSTRUCTION   \n",
       "47683         GENERAL FUND                    NaN  REGULAR INSTRUCTION   \n",
       "229958        GENERAL FUND                    NaN    EMPLOYEE BENEFITS   \n",
       "\n",
       "              Text_2       Text_3               Text_4  \n",
       "134338           NaN          NaN                  NaN  \n",
       "326408           NaN          NaN  Regular Instruction  \n",
       "364634  TEACHER SUBS      Regular                  NaN  \n",
       "47683   TEACHER SUBS  Alternative                  NaN  \n",
       "229958           NaN          n/a                  NaN  "
      ]
     },
     "execution_count": 151,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Function_Aides Compensation</th>\n",
       "      <th>Function_Career &amp; Academic Counseling</th>\n",
       "      <th>Function_Communications</th>\n",
       "      <th>Function_Curriculum Development</th>\n",
       "      <th>Function_Data Processing &amp; Information Services</th>\n",
       "      <th>Function_Development &amp; Fundraising</th>\n",
       "      <th>Function_Enrichment</th>\n",
       "      <th>Function_Extended Time &amp; Tutoring</th>\n",
       "      <th>Function_Facilities &amp; Maintenance</th>\n",
       "      <th>Function_Facilities Planning</th>\n",
       "      <th>...</th>\n",
       "      <th>Object_Type_Rent/Utilities</th>\n",
       "      <th>Object_Type_Substitute Compensation</th>\n",
       "      <th>Object_Type_Supplies/Materials</th>\n",
       "      <th>Object_Type_Travel &amp; Conferences</th>\n",
       "      <th>Pre_K_NO_LABEL</th>\n",
       "      <th>Pre_K_Non PreK</th>\n",
       "      <th>Pre_K_PreK</th>\n",
       "      <th>Operating_Status_Non-Operating</th>\n",
       "      <th>Operating_Status_Operating, Not PreK-12</th>\n",
       "      <th>Operating_Status_PreK-12 Operating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>134338</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>326408</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>364634</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47683</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>229958</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 104 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        Function_Aides Compensation  Function_Career & Academic Counseling  \\\n",
       "134338                            0                                      0   \n",
       "326408                            0                                      0   \n",
       "364634                            0                                      0   \n",
       "47683                             0                                      0   \n",
       "229958                            0                                      0   \n",
       "\n",
       "        Function_Communications  Function_Curriculum Development  \\\n",
       "134338                        0                                0   \n",
       "326408                        0                                0   \n",
       "364634                        0                                0   \n",
       "47683                         0                                0   \n",
       "229958                        0                                0   \n",
       "\n",
       "        Function_Data Processing & Information Services  \\\n",
       "134338                                                0   \n",
       "326408                                                0   \n",
       "364634                                                0   \n",
       "47683                                                 0   \n",
       "229958                                                0   \n",
       "\n",
       "        Function_Development & Fundraising  Function_Enrichment  \\\n",
       "134338                                   0                    0   \n",
       "326408                                   0                    0   \n",
       "364634                                   0                    0   \n",
       "47683                                    0                    0   \n",
       "229958                                   0                    0   \n",
       "\n",
       "        Function_Extended Time & Tutoring  Function_Facilities & Maintenance  \\\n",
       "134338                                  0                                  0   \n",
       "326408                                  0                                  0   \n",
       "364634                                  0                                  0   \n",
       "47683                                   0                                  0   \n",
       "229958                                  0                                  1   \n",
       "\n",
       "        Function_Facilities Planning                 ...                  \\\n",
       "134338                             0                 ...                   \n",
       "326408                             0                 ...                   \n",
       "364634                             0                 ...                   \n",
       "47683                              0                 ...                   \n",
       "229958                             0                 ...                   \n",
       "\n",
       "        Object_Type_Rent/Utilities  Object_Type_Substitute Compensation  \\\n",
       "134338                           0                                    0   \n",
       "326408                           0                                    0   \n",
       "364634                           0                                    0   \n",
       "47683                            0                                    1   \n",
       "229958                           0                                    0   \n",
       "\n",
       "        Object_Type_Supplies/Materials  Object_Type_Travel & Conferences  \\\n",
       "134338                               0                                 0   \n",
       "326408                               0                                 0   \n",
       "364634                               0                                 0   \n",
       "47683                                0                                 0   \n",
       "229958                               0                                 0   \n",
       "\n",
       "        Pre_K_NO_LABEL  Pre_K_Non PreK  Pre_K_PreK  \\\n",
       "134338               1               0           0   \n",
       "326408               0               1           0   \n",
       "364634               1               0           0   \n",
       "47683                1               0           0   \n",
       "229958               1               0           0   \n",
       "\n",
       "        Operating_Status_Non-Operating  \\\n",
       "134338                               0   \n",
       "326408                               0   \n",
       "364634                               0   \n",
       "47683                                0   \n",
       "229958                               0   \n",
       "\n",
       "        Operating_Status_Operating, Not PreK-12  \\\n",
       "134338                                        0   \n",
       "326408                                        0   \n",
       "364634                                        0   \n",
       "47683                                         0   \n",
       "229958                                        0   \n",
       "\n",
       "        Operating_Status_PreK-12 Operating  \n",
       "134338                                   1  \n",
       "326408                                   1  \n",
       "364634                                   1  \n",
       "47683                                    1  \n",
       "229958                                   1  \n",
       "\n",
       "[5 rows x 104 columns]"
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "     steps=[('union', FeatureUnion(n_jobs=1,\n",
       "       transformer_list=[('numeric_features', Pipeline(memory=None,\n",
       "     steps=[('selector', FunctionTransformer(accept_sparse=False,\n",
       "          func=<function <lambda> at 0x12fb00158>, inv_kw_args=None,\n",
       "          inverse_func=None, kw_args=None, pass_y='deprec...=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False),\n",
       "          n_jobs=1))])"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pl.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3695459371681969"
      ]
     },
     "execution_count": 154,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pl.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "     steps=[('union', FeatureUnion(n_jobs=1,\n",
       "       transformer_list=[('numeric_features', Pipeline(memory=None,\n",
       "     steps=[('selector', FunctionTransformer(accept_sparse=False,\n",
       "          func=<function <lambda> at 0x12fb00158>, inv_kw_args=None,\n",
       "          inverse_func=None, kw_args=None, pass_y='deprec...=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False),\n",
       "          n_jobs=1))])"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pl_alphanum.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.35267003934794827"
      ]
     },
     "execution_count": 156,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pl_alphanum.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "     steps=[('union', FeatureUnion(n_jobs=1,\n",
       "       transformer_list=[('numeric_features', Pipeline(memory=None,\n",
       "     steps=[('selector', FunctionTransformer(accept_sparse=False,\n",
       "          func=<function <lambda> at 0x12fb00158>, inv_kw_args=None,\n",
       "          inverse_func=None, kw_args=None, pass_y='deprec...=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False),\n",
       "          n_jobs=1))])"
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pl_basic.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3658609705827244"
      ]
     },
     "execution_count": 158,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pl_basic.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "When no parameter is passed to CountVectorizer, it uses the default value: From sklearn documentation:  The default regexp select tokens of 2 or more alphanumeric characters (punctuation is completely ignored and always treated as a token separator)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's test another model. See how using pipeline makes it all very nifty."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plRF = Pipeline([\n",
    "    ('union', process_and_join_features),\n",
    "    ('clf', OneVsRestClassifier(RandomForestClassifier()))\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "     steps=[('union', FeatureUnion(n_jobs=1,\n",
       "       transformer_list=[('numeric_features', Pipeline(memory=None,\n",
       "     steps=[('selector', FunctionTransformer(accept_sparse=False,\n",
       "          func=<function <lambda> at 0x12fb00158>, inv_kw_args=None,\n",
       "          inverse_func=None, kw_args=None, pass_y='deprec... oob_score=False, random_state=None, verbose=0,\n",
       "            warm_start=False),\n",
       "          n_jobs=1))])"
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "plRF.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8991068640309787"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "plRF.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Wow! Huge improvement on the score."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "Evidently RandomForestClassifier() works with multiclass labels out of the box. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** Using n-grams and MaxAbsScaler **"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import MaxAbsScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pl = Pipeline([\n",
    "        ('union', FeatureUnion(\n",
    "            transformer_list = [\n",
    "                ('numeric_features', Pipeline([\n",
    "                    ('selector', get_numeric_data),\n",
    "                    ('imputer', Imputer())\n",
    "                ])),\n",
    "                ('text_features', Pipeline([\n",
    "                    ('selector', get_text_data),\n",
    "                    ('vectorizer', CountVectorizer(token_pattern=TOKENS_ALPHANUMERIC,\n",
    "                                                   ngram_range=(1,2)))\n",
    "                ]))\n",
    "             ]\n",
    "        )),\n",
    "        ('scale', MaxAbsScaler()),\n",
    "        ('clf', OneVsRestClassifier(LogisticRegression()))\n",
    "    ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "     steps=[('union', FeatureUnion(n_jobs=1,\n",
       "       transformer_list=[('numeric_features', Pipeline(memory=None,\n",
       "     steps=[('selector', FunctionTransformer(accept_sparse=False,\n",
       "          func=<function <lambda> at 0x12fb00158>, inv_kw_args=None,\n",
       "          inverse_func=None, kw_args=None, pass_y='deprec...=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False),\n",
       "          n_jobs=1))])"
      ]
     },
     "execution_count": 169,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pl.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9202298419836362"
      ]
     },
     "execution_count": 170,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pl.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "SparseInteractions code from https://github.com/drivendataorg/box-plots-sklearn/blob/master/src/features/SparseInteractions.py\n",
    "\n",
    "SparseInteractions does the same thing as sklearn.preprocessing.PolynomialFeatures, but it uses sparse matrices to do so."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from itertools import combinations\n",
    "\n",
    "import numpy as np\n",
    "from scipy import sparse\n",
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "\n",
    "\n",
    "class SparseInteractions(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, degree=2, feature_name_separator=\"_\"):\n",
    "        self.degree = degree\n",
    "        self.feature_name_separator = feature_name_separator\n",
    "\n",
    "    def fit(self, X, y=None):\n",
    "        return self\n",
    "\n",
    "    def transform(self, X):\n",
    "        if not sparse.isspmatrix_csc(X):\n",
    "            X = sparse.csc_matrix(X)\n",
    "\n",
    "        if hasattr(X, \"columns\"):\n",
    "            self.orig_col_names = X.columns\n",
    "        else:\n",
    "            self.orig_col_names = np.array([str(i) for i in range(X.shape[1])])\n",
    "\n",
    "        spi = self._create_sparse_interactions(X)\n",
    "        return spi\n",
    "\n",
    "    def get_feature_names(self):\n",
    "        return self.feature_names\n",
    "\n",
    "    def _create_sparse_interactions(self, X):\n",
    "        out_mat = []\n",
    "        self.feature_names = self.orig_col_names.tolist()\n",
    "\n",
    "        for sub_degree in range(2, self.degree + 1):\n",
    "            for col_ixs in combinations(range(X.shape[1]), sub_degree):\n",
    "                # add name for new column\n",
    "                name = self.feature_name_separator.join(self.orig_col_names[list(col_ixs)])\n",
    "                self.feature_names.append(name)\n",
    "\n",
    "                # get column multiplications value\n",
    "                out = X[:, col_ixs[0]]\n",
    "                for j in col_ixs[1:]:\n",
    "                    out = out.multiply(X[:, j])\n",
    "\n",
    "                out_mat.append(out)\n",
    "\n",
    "        return sparse.hstack([X] + out_mat)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Implementing the interaction model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [],
   "source": [
    "pl = Pipeline([\n",
    "        ('union', FeatureUnion(\n",
    "            transformer_list = [\n",
    "                ('numeric_features', Pipeline([\n",
    "                    ('selector', get_numeric_data),\n",
    "                    ('imputer', Imputer())\n",
    "                ])),\n",
    "                ('text_features', Pipeline([\n",
    "                    ('selector', get_text_data),\n",
    "                    ('vectorizer', CountVectorizer(token_pattern=TOKENS_ALPHANUMERIC,\n",
    "                                                   ngram_range=(1, 2)))\n",
    "                ]))\n",
    "             ]\n",
    "        )),\n",
    "        ('int', SparseInteractions(degree=2)),\n",
    "        ('scale', MaxAbsScaler()),\n",
    "        ('clf', OneVsRestClassifier(LogisticRegression()))\n",
    "    ])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using HashingVectorizer to speed things up"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Import HashingVectorizer\n",
    "from sklearn.feature_extraction.text import HashingVectorizer\n",
    "\n",
    "# Get text data: text_data\n",
    "text_data = combine_text_columns(X_train)\n",
    "\n",
    "# Create the token pattern: TOKENS_ALPHANUMERIC\n",
    "TOKENS_ALPHANUMERIC = '[A-Za-z0-9]+(?=\\\\s+)' \n",
    "\n",
    "# Instantiate the HashingVectorizer: hashing_vec\n",
    "hashing_vec = HashingVectorizer(token_pattern = TOKENS_ALPHANUMERIC)\n",
    "\n",
    "# Fit and transform the Hashing Vectorizer\n",
    "hashed_text = hashing_vec.fit_transform(text_data)\n",
    "\n",
    "# Create DataFrame and print the head\n",
    "hashed_df = pd.DataFrame(hashed_text.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<320222x1048576 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 4658891 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 176,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hashed_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.37796447,  0.75592895,  0.37796447, ..., -0.12598816,\n",
       "       -0.12598816, -0.12598816])"
      ]
     },
     "execution_count": 177,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hashed_text.data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.377964</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.755929</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.377964</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.377964</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.258199</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          0\n",
       "0  0.377964\n",
       "1  0.755929\n",
       "2  0.377964\n",
       "3  0.377964\n",
       "4  0.258199"
      ]
     },
     "execution_count": 178,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hashed_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "134338      General Fund Teacher-Elementary    KINDERGAR...\n",
       "326408      General Purpose School TCHER 2ND GRADE  Pers...\n",
       "364634     UNALLOC BUDGETS/SCHOOLS  Teacher, Short Term ...\n",
       "47683      NON-PROJECT  Teacher, Secondary (High)   TEAC...\n",
       "229958     NON-PROJECT  Custodian - PT -  Jobs   CONTRA ...\n",
       "dtype: object"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}