notebooks/10_advanced_sklearn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.pipeline import Pipeline, FeatureUnion, make_pipeline, make_union\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.feature_selection import SelectKBest\n",
    "from sklearn.datasets import load_iris\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.cross_validation import cross_val_score\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.base import TransformerMixin\n",
    "import pandas as pd\n",
    "from sklearn.cross_validation import train_test_split\n",
    "from sklearn import metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Some text\n",
    "\n",
    "X = (\n",
    "    \"the pizza pizza beer copyright\",\n",
    "    \"the pizza burger beer copyright\",\n",
    "    \"the the pizza beer beer copyright\",\n",
    "    \"the burger beer beer copyright\",\n",
    "    \"the coke burger coke copyright\",\n",
    "    \"the coke burger burger\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# EXERCISE (WAKE UP ITS CODING TIME)\n",
    "\n",
    "# Use a count vectorizer to count vectorize X and show a \n",
    "# user-friendly dataframe to see the count of words in each sentence\n",
    "# Recall this is otherwise known as a document term matrix, make it\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>beer</th>\n",
       "      <th>burger</th>\n",
       "      <th>coke</th>\n",
       "      <th>copyright</th>\n",
       "      <th>pizza</th>\n",
       "      <th>the</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   beer  burger  coke  copyright  pizza  the\n",
       "0     1       0     0          1      2    1\n",
       "1     1       1     0          1      1    1\n",
       "2     2       0     0          1      1    2\n",
       "3     2       1     0          1      0    1\n",
       "4     0       1     2          1      0    1\n",
       "5     0       2     1          0      0    1"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# ANSWER\n",
    "\n",
    "word_vect = CountVectorizer(analyzer='word')\n",
    "words = pd.DataFrame(word_vect.fit_transform(X).toarray(), columns=word_vect.get_feature_names())\n",
    "words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>e</th>\n",
       "      <th>g</th>\n",
       "      <th>h</th>\n",
       "      <th>i</th>\n",
       "      <th>k</th>\n",
       "      <th>o</th>\n",
       "      <th>p</th>\n",
       "      <th>r</th>\n",
       "      <th>t</th>\n",
       "      <th>u</th>\n",
       "      <th>y</th>\n",
       "      <th>z</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      a  b  c  e  g  h  i  k  o  p  r  t  u  y  z\n",
       "0  4  2  1  1  3  1  2  3  0  1  3  2  2  0  1  4\n",
       "1  4  1  2  1  4  2  2  2  0  1  2  4  2  1  1  2\n",
       "2  5  1  2  1  6  1  3  2  0  1  2  3  3  0  1  2\n",
       "3  4  0  3  1  6  2  2  1  0  1  1  5  2  1  1  0\n",
       "4  4  0  1  3  4  2  2  1  2  3  1  3  2  1  1  0\n",
       "5  3  0  2  1  4  2  1  0  1  1  0  4  1  2  0  0"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "char_vect = CountVectorizer(analyzer='char')\n",
    "chars = pd.DataFrame(char_vect.fit_transform(X).toarray(), columns=char_vect.get_feature_names())\n",
    "chars"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# This is a feature union. If automatically horizontally stacks our count and tfidf vectorizers\n",
    "# http://scikit-learn.org/stable/modules/generated/sklearn.pipeline.FeatureUnion.html\n",
    "fs = FeatureUnion([\n",
    "    (\"words\", CountVectorizer(analyzer='word')),\n",
    "    (\"chars\", CountVectorizer(analyzer='char')),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "FeatureUnion(n_jobs=1,\n",
       "       transformer_list=[('words', CountVectorizer(analyzer='word', binary=False, decode_error=u'strict',\n",
       "        dtype=<type 'numpy.int64'>, encoding=u'utf-8', input=u'content',\n",
       "        lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
       "        ngram_range=(1, 1), preprocessor=None, stop_words=None,...  strip_accents=None, token_pattern=u'(?u)\\\\b\\\\w\\\\w+\\\\b',\n",
       "        tokenizer=None, vocabulary=None))],\n",
       "       transformer_weights=None)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fs #see our params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>beer</th>\n",
       "      <th>burger</th>\n",
       "      <th>coke</th>\n",
       "      <th>copyright</th>\n",
       "      <th>pizza</th>\n",
       "      <th>the</th>\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>...</th>\n",
       "      <th>h</th>\n",
       "      <th>i</th>\n",
       "      <th>k</th>\n",
       "      <th>o</th>\n",
       "      <th>p</th>\n",
       "      <th>r</th>\n",
       "      <th>t</th>\n",
       "      <th>u</th>\n",
       "      <th>y</th>\n",
       "      <th>z</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   beer  burger  coke  copyright  pizza  the     a  b  c ...  h  i  k  o  p  \\\n",
       "0     1       0     0          1      2    1  4  2  1  1 ...  2  3  0  1  3   \n",
       "1     1       1     0          1      1    1  4  1  2  1 ...  2  2  0  1  2   \n",
       "2     2       0     0          1      1    2  5  1  2  1 ...  3  2  0  1  2   \n",
       "3     2       1     0          1      0    1  4  0  3  1 ...  2  1  0  1  1   \n",
       "4     0       1     2          1      0    1  4  0  1  3 ...  2  1  2  3  1   \n",
       "5     0       2     1          0      0    1  3  0  2  1 ...  1  0  1  1  0   \n",
       "\n",
       "   r  t  u  y  z  \n",
       "0  2  2  0  1  4  \n",
       "1  4  2  1  1  2  \n",
       "2  3  3  0  1  2  \n",
       "3  5  2  1  1  0  \n",
       "4  3  2  1  1  0  \n",
       "5  4  1  2  0  0  \n",
       "\n",
       "[6 rows x 22 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# To see them put together\n",
    "\n",
    "words_plus_chars = pd.DataFrame(fs.fit_transform(X).toarray(), columns=word_vect.get_feature_names()+char_vect.get_feature_names())\n",
    "words_plus_chars"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(6, 6)\n",
      "+\n",
      "(6, 16)\n",
      "========\n",
      "(6, 22)\n"
     ]
    }
   ],
   "source": [
    "print words.shape\n",
    "print \"+\"\n",
    "print chars.shape\n",
    "print \"========\"\n",
    "print words_plus_chars.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Pipeline http://scikit-learn.org/stable/modules/generated/sklearn.pipeline.Pipeline.html\n",
    "# is a module that chains different sklearn modules together in order"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Test the various methods of the pipeline (anova).\n",
    "iris = load_iris()\n",
    "X = iris.data\n",
    "y = iris.target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.85999999999999999"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Test with selectkbest + LogisticRegression\n",
    "# SelectKBest http://scikit-learn.org/stable/modules/feature_selection.html#univariate-feature-selection\n",
    "# will automatically attempt to reduce the number of features by performing \n",
    "# statistical tests (including ANOVA or Chi Squared)\n",
    "\n",
    "logreg = LogisticRegression()\n",
    "filter1 = SelectKBest(k=2) # select the best 2 features\n",
    "pipe = Pipeline([('anova', filter1), ('logistic', logreg)])\n",
    "\n",
    "\n",
    "mean = cross_val_score(pipe, X, y, cv=10, scoring='accuracy').mean()\n",
    "mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.85999999999999999"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# You can use make_pipeline to make simpler pipelines\n",
    "easy_pipe = make_pipeline(SelectKBest(k=2), LogisticRegression())\n",
    "mean = cross_val_score(easy_pipe, X, y, cv=10, scoring='accuracy').mean()\n",
    "mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>msg</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ham</td>\n",
       "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ham</td>\n",
       "      <td>Ok lar... Joking wif u oni...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>spam</td>\n",
       "      <td>Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ham</td>\n",
       "      <td>U dun say so early hor... U c already then say...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ham</td>\n",
       "      <td>Nah I don't think he goes to usf, he lives aro...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  label                                                msg\n",
       "0   ham  Go until jurong point, crazy.. Available only ...\n",
       "1   ham                      Ok lar... Joking wif u oni...\n",
       "2  spam  Free entry in 2 a wkly comp to win FA Cup fina...\n",
       "3   ham  U dun say so early hor... U c already then say...\n",
       "4   ham  Nah I don't think he goes to usf, he lives aro..."
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_table('../data/sms.tsv',\n",
    "                   sep='\\t', header=None, names=['label', 'msg'])\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# EXERCISE\n",
    "\n",
    "\n",
    "# cross validate two naive bayes models.\n",
    "# One model should attempt to fit to the count vectorized text\n",
    "# The other should fit to the tfidfed text\n",
    "# Each model whould be fit over a 10 fold validation with accuracy\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.98026032061360824"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "# Answers\n",
    "\n",
    "vect = CountVectorizer()\n",
    "all_dtm = vect.fit_transform(df.msg)\n",
    "nb = MultinomialNB()\n",
    "mean = cross_val_score(nb, all_dtm, df.label, cv=10, scoring='accuracy').mean()\n",
    "mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.96320881558493299"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vect = TfidfVectorizer()\n",
    "all_dtm = vect.fit_transform(df.msg)\n",
    "nb = MultinomialNB()\n",
    "mean = cross_val_score(nb, all_dtm, df.label, cv=10, scoring='accuracy').mean()\n",
    "mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9840269835443396"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# A more complicated pipeline\n",
    "\n",
    "pipe = Pipeline([\n",
    "  ('features', FeatureUnion([\n",
    "        ('counts', CountVectorizer()),\n",
    "        ('tf_idf', TfidfVectorizer())\n",
    "  ])),\n",
    "  ('classifier', MultinomialNB())\n",
    "])\n",
    "mean = cross_val_score(pipe, df.msg, df.label, cv=10, scoring='accuracy').mean()\n",
    "mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Not only is this PROPER cross validation of count vectorizers, its better than either model combined!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'features__counts__ngram_range': [(1, 1), (1, 2)], 'features__tf_idf__ngram_range': [(1, 1), (1, 2)]}\n",
      "\n",
      "0.985642498205 {'features__counts__ngram_range': (1, 2), 'features__tf_idf__ngram_range': (1, 1)}\n"
     ]
    }
   ],
   "source": [
    "# Pipeline can also be used with GridSearchCV, WARNING SLOWWW\n",
    "pipe = Pipeline([\n",
    "  ('features', FeatureUnion([\n",
    "        ('counts', CountVectorizer()),\n",
    "        ('tf_idf', TfidfVectorizer())\n",
    "  ])),\n",
    "  ('classifier', MultinomialNB())\n",
    "])\n",
    "# search for an optimal n_neighbors value using GridSearchCV\n",
    "gram_range = [(1, n) for n in range(1, 3)]\n",
    "param_grid = {\n",
    "    'features__counts__ngram_range': gram_range,\n",
    "    'features__tf_idf__ngram_range': gram_range,\n",
    "}\n",
    "print param_grid\n",
    "print\n",
    "grid = GridSearchCV(pipe, param_grid, cv=5, scoring='accuracy')\n",
    "grid.fit(df.msg, df.label)\n",
    "print grid.best_score_, grid.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# We can also make our own custom transformations!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class DatetimeTransformer(TransformerMixin):\n",
    "    def transform(self, X, **transform_params): \n",
    "        if 'date' in X:\n",
    "            years = X['date'].apply(lambda x: x.year)\n",
    "            months = X['date'].apply(lambda x: x.month)\n",
    "            days = X['date'].apply(lambda x: x.day)\n",
    "            return pd.DataFrame({'year':years, 'month':months, 'day':days})\n",
    "        return X\n",
    "\n",
    "    def fit(self, X, y=None, **fit_params):\n",
    "        return self"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>stars</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2011-01-26</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2011-07-27</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2010-05-27</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2012-01-05</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2010-02-12</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        date  stars\n",
       "0 2011-01-26      5\n",
       "1 2011-07-27      5\n",
       "3 2010-05-27      5\n",
       "4 2012-01-05      5\n",
       "6 2010-02-12      5"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url = '../data/yelp.csv'\n",
    "yelp = pd.read_csv(url, encoding='unicode-escape', usecols=[1, 3])\n",
    "yelp['date'] = pd.to_datetime(yelp['date'])\n",
    "yelp = yelp[(yelp.stars == 1) | (yelp.stars == 5)] # only look at best and worst ratings\n",
    "yelp.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>day</th>\n",
       "      <th>month</th>\n",
       "      <th>year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>26</td>\n",
       "      <td>1</td>\n",
       "      <td>2011</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>27</td>\n",
       "      <td>7</td>\n",
       "      <td>2011</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>27</td>\n",
       "      <td>5</td>\n",
       "      <td>2010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>2012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>12</td>\n",
       "      <td>2</td>\n",
       "      <td>2010</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   day  month  year\n",
       "0   26      1  2011\n",
       "1   27      7  2011\n",
       "3   27      5  2010\n",
       "4    5      1  2012\n",
       "6   12      2  2010"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "h = DatetimeTransformer() # our custom transformer\n",
    "h.fit_transform(yelp).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>stars</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2011-01-26</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2011-07-27</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2010-05-27</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2012-01-05</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2010-02-12</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        date  stars\n",
       "0 2011-01-26      5\n",
       "1 2011-07-27      5\n",
       "3 2010-05-27      5\n",
       "4 2012-01-05      5\n",
       "6 2010-02-12      5"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "yelp.head() # original data unscathed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.81669161774511989"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# pipeline with our custom transformer and logistic regression\n",
    "\n",
    "pipe = Pipeline([('transform_datetime', DatetimeTransformer()), ('logistic', logreg)])\n",
    "mean = cross_val_score(pipe, yelp.drop('stars', axis=1), yelp.stars, cv=10, scoring='accuracy').mean()\n",
    "mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Hmmmmmm not bad... but what was that null accuracy?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5    0.816691\n",
       "1    0.183309\n",
       "Name: stars, dtype: float64"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "yelp['stars'].value_counts() / yelp.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# It appears dates aren't helping at all.."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>color</th>\n",
       "      <th>proline</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>5.64</td>\n",
       "      <td>1065</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>4.38</td>\n",
       "      <td>1050</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>5.68</td>\n",
       "      <td>1185</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>7.80</td>\n",
       "      <td>1480</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>4.32</td>\n",
       "      <td>735</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   label  color  proline\n",
       "0      1   5.64     1065\n",
       "1      1   4.38     1050\n",
       "2      1   5.68     1185\n",
       "3      1   7.80     1480\n",
       "4      1   4.32      735"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# read three columns from the dataset into a DataFrame\n",
    "url = 'http://archive.ics.uci.edu/ml/machine-learning-databases/wine/wine.data'\n",
    "col_names = ['label', 'color', 'proline']\n",
    "wine = pd.read_csv(url, header=None, names=col_names, usecols=[0, 10, 13])\n",
    "wine.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>color</th>\n",
       "      <th>proline</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>178.000000</td>\n",
       "      <td>178.000000</td>\n",
       "      <td>178.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.938202</td>\n",
       "      <td>5.058090</td>\n",
       "      <td>746.893258</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.775035</td>\n",
       "      <td>2.318286</td>\n",
       "      <td>314.907474</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.280000</td>\n",
       "      <td>278.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.220000</td>\n",
       "      <td>500.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>2.000000</td>\n",
       "      <td>4.690000</td>\n",
       "      <td>673.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>6.200000</td>\n",
       "      <td>985.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>13.000000</td>\n",
       "      <td>1680.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            label       color      proline\n",
       "count  178.000000  178.000000   178.000000\n",
       "mean     1.938202    5.058090   746.893258\n",
       "std      0.775035    2.318286   314.907474\n",
       "min      1.000000    1.280000   278.000000\n",
       "25%      1.000000    3.220000   500.500000\n",
       "50%      2.000000    4.690000   673.500000\n",
       "75%      3.000000    6.200000   985.000000\n",
       "max      3.000000   13.000000  1680.000000"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wine.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# define X and y\n",
    "feature_cols = ['color', 'proline']\n",
    "X = wine[feature_cols]\n",
    "y = wine.label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-3.90664944003e-16\n",
      "1.0\n",
      "1.6027279754e-16\n",
      "1.0\n"
     ]
    }
   ],
   "source": [
    "# split into training and testing sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=1)\n",
    "\n",
    "\n",
    "# standardize X_train\n",
    "scaler = StandardScaler()\n",
    "scaler.fit(X_train)\n",
    "X_train_scaled = scaler.transform(X_train)\n",
    "\n",
    "\n",
    "# Recall that standard scalar does a basic z-score normalization\n",
    "# So for each element, x, it computes z = (x - mean) / std\n",
    "# This gives the entire series a mean of 0 and a standard deviation (and variance) of 1\n",
    "\n",
    "# check that it standardized properly\n",
    "\n",
    "# First column of X_train_scaled\n",
    "print X_train_scaled[:, 0].mean() # 0\n",
    "print X_train_scaled[:, 0].std()  # 1\n",
    "\n",
    "# Second column of X_train_scaled\n",
    "print X_train_scaled[:, 1].mean() # 0\n",
    "print X_train_scaled[:, 1].std()  # 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# standardize X_test\n",
    "X_test_scaled = scaler.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0305898576303\n",
      "0.866822198488\n",
      "0.0546533341088\n",
      "1.14955947533\n"
     ]
    }
   ],
   "source": [
    "# Is anything wrong here?\n",
    "\n",
    "# First column of X_train_scaled\n",
    "print X_test_scaled[:, 0].mean()\n",
    "print X_test_scaled[:, 0].std()\n",
    "\n",
    "\n",
    "# Second column of X_train_scaled\n",
    "print X_test_scaled[:, 1].mean()\n",
    "print X_test_scaled[:, 1].std()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# EXERCISE\n",
    "\n",
    "# Run a KNN (3 neighbors) on the training set and find the accuracy on the testing set\n",
    "# Do this TWICE once for the unscaled data, and once for the scaled data\n",
    "# What do you notice about the accuracies\n",
    "\n",
    "\n",
    "# Thought experiment: Why am I not asking for a cross validation?\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.644444444444\n"
     ]
    }
   ],
   "source": [
    "# Answers\n",
    "\n",
    "# KNN accuracy on original data\n",
    "knn = KNeighborsClassifier(n_neighbors=3)\n",
    "knn.fit(X_train, y_train)\n",
    "y_pred_class = knn.predict(X_test)\n",
    "print metrics.accuracy_score(y_test, y_pred_class)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.866666666667\n"
     ]
    }
   ],
   "source": [
    "# KNN accuracy on scaled data\n",
    "knn.fit(X_train_scaled, y_train)\n",
    "y_pred_class = knn.predict(X_test_scaled)\n",
    "print metrics.accuracy_score(y_test, y_pred_class)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# So we agree, scaling data helps us out in KNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# EXERCISE\n",
    "\n",
    "\n",
    "# Cross validate a KNN with 3 neighbors on the X and y variables with  (UNSCALED)\n",
    "# 5 folds and output average accuracy for the models\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.71983168041991563"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# ANSWER proper cross-validation on the original (unscaled) data\n",
    "\n",
    "knn = KNeighborsClassifier(n_neighbors=3)\n",
    "cross_val_score(knn, X, y, cv=5, scoring='accuracy').mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.90104247104247115"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Why is this improper cross-validation on the scaled data?\n",
    "\n",
    "scaler = StandardScaler()\n",
    "X_scaled = scaler.fit_transform(X)\n",
    "cross_val_score(knn, X_scaled, y, cv=5, scoring='accuracy').mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# EXERCISE\n",
    "\n",
    "# Make a pipeline that scales data and then runs a KNN with 3 neighbors\n",
    "# cross validate your pipeline using cv=5 with accuacy metric\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.89516011810129448"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# ANSWER\n",
    "\n",
    "\n",
    "# fix the cross-validation process using Pipeline\n",
    "pipe = make_pipeline(StandardScaler(), KNeighborsClassifier(n_neighbors=3))\n",
    "cross_val_score(pipe, X, y, cv=5, scoring='accuracy').mean()\n",
    "\n",
    "# It is NOT as good... But it is more accurate to our true estimate (and also better than the unscaled data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.910112359551 {'kneighborsclassifier__n_neighbors': 1}\n"
     ]
    }
   ],
   "source": [
    "# search for an optimal n_neighbors value using GridSearchCV\n",
    "pipe = make_pipeline(StandardScaler(), KNeighborsClassifier())\n",
    "neighbors_range = range(1, 21)\n",
    "param_grid = dict(kneighborsclassifier__n_neighbors=neighbors_range)\n",
    "grid = GridSearchCV(pipe, param_grid, cv=5, scoring='accuracy')\n",
    "grid.fit(X, y)\n",
    "print grid.best_score_, grid.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [sfdat28]",
   "language": "python",
   "name": "Python [sfdat28]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}