notebooks/pipeline.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.pipeline import make_pipeline, Pipeline, FeatureUnion\n",
    "from sklearn.cross_validation import cross_val_score\n",
    "from textblob import TextBlob\n",
    "from sklearn.naive_bayes import MultinomialNB"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Why do we make these pipelines?\n",
    "\n",
    "## Let's start with why we don't..\n",
    "* We don't user pipelines to chain together \"normal\" non sklearn functions\n",
    "    * pipe = make_pipeline(lambda x:x+1, lambda x:x*2) # does not work"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>color</th>\n",
       "      <th>proline</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td> 1</td>\n",
       "      <td> 5.64</td>\n",
       "      <td> 1065</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td> 1</td>\n",
       "      <td> 4.38</td>\n",
       "      <td> 1050</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td> 1</td>\n",
       "      <td> 5.68</td>\n",
       "      <td> 1185</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td> 1</td>\n",
       "      <td> 7.80</td>\n",
       "      <td> 1480</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td> 1</td>\n",
       "      <td> 4.32</td>\n",
       "      <td>  735</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   label  color  proline\n",
       "0      1   5.64     1065\n",
       "1      1   4.38     1050\n",
       "2      1   5.68     1185\n",
       "3      1   7.80     1480\n",
       "4      1   4.32      735"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url = 'http://archive.ics.uci.edu/ml/machine-learning-databases/wine/wine.data'\n",
    "col_names = ['label', 'color', 'proline']\n",
    "wine = pd.read_csv(url, header=None, names=col_names, usecols=[0, 10, 13])\n",
    "wine.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# define X and y\n",
    "feature_cols = ['color', 'proline']\n",
    "X = wine[feature_cols]\n",
    "y = wine.label"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Why we use Pipelines\n",
    "* For cross validation modularization, what??\n",
    "* for gridsearching (for the same reason)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.90104247104247115"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# define X and y\n",
    "feature_cols = ['color', 'proline']\n",
    "X = wine[feature_cols]\n",
    "y = wine.label\n",
    "\n",
    "\n",
    "# improper cross-validation on the scaled data. Why is that?\n",
    "scaler = StandardScaler()\n",
    "X_scaled = scaler.fit_transform(X)\n",
    "cross_val_score(knn, X_scaled, y, cv=5, scoring='accuracy').mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.89516011810129448"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# proper cross validation\n",
    "pipe = make_pipeline(StandardScaler(), KNeighborsClassifier(n_neighbors=3))\n",
    "cross_val_score(pipe, X, y, cv=5, scoring='accuracy').mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('standardscaler', StandardScaler(copy=True, with_mean=True, with_std=True)),\n",
       " ('kneighborsclassifier',\n",
       "  KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
       "             metric_params=None, n_neighbors=3, p=2, weights='uniform'))]"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipe.steps # see the steps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'kneighborsclassifier': KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
       "            metric_params=None, n_neighbors=3, p=2, weights='uniform'),\n",
       " 'standardscaler': StandardScaler(copy=True, with_mean=True, with_std=True)}"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipe.named_steps #see the steps with names, useful for grid searching"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.910112359551\n",
      "{'kneighborsclassifier__n_neighbors': 1}\n"
     ]
    }
   ],
   "source": [
    "# search for an optimal n_neighbors value using GridSearchCV\n",
    "neighbors_range = range(1, 31)\n",
    "param_grid = dict(kneighborsclassifier__n_neighbors=neighbors_range)\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "grid = GridSearchCV(pipe, param_grid, cv=5, scoring='accuracy')\n",
    "grid.fit(X, y)\n",
    "print grid.best_score_\n",
    "print grid.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Note the nomenclauture for the parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2])"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "optimal_model = KNeighborsClassifier(grid.best_params_['kneighborsclassifier__n_neighbors'])\n",
    "optimal_model.fit(X, y)\n",
    "optimal_model.predict([4, 4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Let's play with some spam/ham data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>msg</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>  ham</td>\n",
       "      <td> Go until jurong point, crazy.. Available only ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>  ham</td>\n",
       "      <td>                     Ok lar... Joking wif u oni...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td> spam</td>\n",
       "      <td> Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>  ham</td>\n",
       "      <td> U dun say so early hor... U c already then say...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>  ham</td>\n",
       "      <td> Nah I don't think he goes to usf, he lives aro...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  label                                                msg\n",
       "0   ham  Go until jurong point, crazy.. Available only ...\n",
       "1   ham                      Ok lar... Joking wif u oni...\n",
       "2  spam  Free entry in 2 a wkly comp to win FA Cup fina...\n",
       "3   ham  U dun say so early hor... U c already then say...\n",
       "4   ham  Nah I don't think he goes to usf, he lives aro..."
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer, TfidfVectorizer\n",
    "vec = CountVectorizer(ngram_range=(1,5))\n",
    "df = pd.read_table('https://raw.githubusercontent.com/sinanuozdemir/DAT6/master/data/sms.tsv',\n",
    "                   sep='\\t', header=None, names=['label', 'msg'])\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<5572x207807 sparse matrix of type '<type 'numpy.int64'>'\n",
       "\twith 339886 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vec.fit_transform(df['msg'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pipe = make_pipeline(CountVectorizer(ngram_range=(1,5)), KNeighborsClassifier(n_neighbors=3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.90864995853829367"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cross_val_score(pipe, df['msg'], df['label'], cv=5, scoring='accuracy').mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Let's add a new analyzer by lemmatizing everyting\n",
    "def split_into_lemmas(message):\n",
    "    message = unicode(message, 'utf8').lower()\n",
    "    words = TextBlob(message).words\n",
    "    # for each word, take its \"base form\" = lemma \n",
    "    return [word.lemma for word in words]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[u'hello', u'there', u'that', u'is', u'a', u'pack', u'of', u'wolf']"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "split_into_lemmas(\"hello there that is a pack of wolves\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pipe = make_pipeline(CountVectorizer(ngram_range=(1,5),analyzer=split_into_lemmas), KNeighborsClassifier(n_neighbors=3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.92964809879962329"
      ]
     },
     "execution_count": 139,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cross_val_score(pipe, df['msg'], df['label'], cv=5, scoring='accuracy').mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'countvectorizer': CountVectorizer(analyzer=u'word', binary=False, decode_error=u'strict',\n",
      "        dtype=<type 'numpy.int64'>, encoding=u'utf-8', input=u'content',\n",
      "        lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
      "        ngram_range=(1, 1), preprocessor=None, stop_words=None,\n",
      "        strip_accents=None, token_pattern=u'(?u)\\\\b\\\\w\\\\w+\\\\b',\n",
      "        tokenizer=None, vocabulary=None), 'kneighborsclassifier': KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
      "           metric_params=None, n_neighbors=5, p=2, weights='uniform')}\n",
      "{'countvectorizer__ngram_range': [(1, 1), (1, 2)], 'kneighborsclassifier__n_neighbors': [1, 2]}\n",
      "0.949030868629\n",
      "{'countvectorizer__ngram_range': (1, 1), 'kneighborsclassifier__n_neighbors': 1}\n"
     ]
    }
   ],
   "source": [
    "# Exercise\n",
    "\n",
    "# grid search using a pipeline of countvectorizer and KNN and find the optimal combination of neighbors and ngram ranges\n",
    "# et, (1, 2), (1, 3), etc\n",
    "\n",
    "# search for an optimal n_neighbors value using GridSearchCV\n",
    "pipe = make_pipeline(CountVectorizer(), KNeighborsClassifier())\n",
    "print pipe.named_steps\n",
    "param_grid = dict(kneighborsclassifier__n_neighbors=**replace me**, countvectorizer__ngram_range=**replace me**)\n",
    "print param_grid\n",
    "grid = GridSearchCV(**replace me**, param_grid, cv=5, scoring='accuracy')\n",
    "grid.fit(df['msg'], **replace me**)\n",
    "print grid.best_score_\n",
    "print grid.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# A bit more complicated..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pipeline = Pipeline([\n",
    "  ('counts', CountVectorizer()),\n",
    "  ('tf_idf', TfidfTransformer()),\n",
    "  ('classifier', MultinomialNB())\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.95800420252634633"
      ]
     },
     "execution_count": 167,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cross_val_score(pipeline, df['msg'], df['label'], cv=5, scoring='accuracy').mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "NotFittedError",
     "evalue": "CountVectorizer - Vocabulary wasn't fitted.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNotFittedError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-142-3a7654a32c4d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mpipeline\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'hello'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# Won't work!!!! becaue it is trying to FIT to data first\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;31m# must think of pipeline as its own model\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;31m# It can be pickled..\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/sinanozdemir/Library/Python/2.7/lib/python/site-packages/sklearn/utils/metaestimators.pyc\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     35\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_attribute\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     36\u001b[0m         \u001b[0;31m# lambda, but not partial, allows help() to work with update_wrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 37\u001b[0;31m         \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     38\u001b[0m         \u001b[0;31m# update the docstring of the returned function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     39\u001b[0m         \u001b[0mupdate_wrapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/sinanozdemir/Library/Python/2.7/lib/python/site-packages/sklearn/pipeline.pyc\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, X)\u001b[0m\n\u001b[1;32m    177\u001b[0m         \u001b[0mXt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    178\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtransform\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msteps\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 179\u001b[0;31m             \u001b[0mXt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtransform\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mXt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    180\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msteps\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mXt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    181\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/sinanozdemir/Library/Python/2.7/lib/python/site-packages/sklearn/feature_extraction/text.pyc\u001b[0m in \u001b[0;36mtransform\u001b[0;34m(self, raw_documents)\u001b[0m\n\u001b[1;32m    848\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_validate_vocabulary\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    849\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 850\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_check_vocabulary\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    851\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    852\u001b[0m         \u001b[0;31m# use the same matrix-building strategy as fit_transform\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/sinanozdemir/Library/Python/2.7/lib/python/site-packages/sklearn/feature_extraction/text.pyc\u001b[0m in \u001b[0;36m_check_vocabulary\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    269\u001b[0m         \u001b[0;34m\"\"\"Check if vocabulary is empty or missing (not fit-ed)\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    270\u001b[0m         \u001b[0mmsg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"%(name)s - Vocabulary wasn't fitted.\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 271\u001b[0;31m         \u001b[0mcheck_is_fitted\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'vocabulary_'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmsg\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    272\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    273\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvocabulary_\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/sinanozdemir/Library/Python/2.7/lib/python/site-packages/sklearn/utils/validation.pyc\u001b[0m in \u001b[0;36mcheck_is_fitted\u001b[0;34m(estimator, attributes, msg, all_or_any)\u001b[0m\n\u001b[1;32m    620\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    621\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mall_or_any\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mestimator\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattr\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mattr\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mattributes\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 622\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0mNotFittedError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m'name'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mestimator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNotFittedError\u001b[0m: CountVectorizer - Vocabulary wasn't fitted."
     ]
    }
   ],
   "source": [
    "pipeline.predict('hello') # Won't work!!!! becaue it is trying to FIT to data first\n",
    "# must think of pipeline as its own model\n",
    "# It can be pickled.."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Exercise, complete the code below that will\n",
    "# 1. countvectorize df['msg']\n",
    "# 2. tfidf transform the data\n",
    "# 3. fit a naive bayes to the entire data set\n",
    "vec = CountVectorizer()\n",
    "vec.fit(df['msg'])\n",
    "t = TfidfTransformer()\n",
    "tfidfed = t.fit_transform(**replace me**)\n",
    "nb = MultinomialNB()\n",
    "bayes = nb.fit(**replace me**, **replace me**)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def megaPredict(text):\n",
    "    return bayes.predict(**replace me**)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['spam', 'ham'], \n",
       "      dtype='|S4')"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "megaPredict(['100 dating service cal;l 09064012103 box334sk38ch', \"Hey ralph, you coming tonight?\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('counts', CountVectorizer(analyzer=u'word', binary=False, decode_error=u'strict',\n",
       "        dtype=<type 'numpy.int64'>, encoding=u'utf-8', input=u'content',\n",
       "        lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
       "        ngram_range=(1, 1), preprocessor=None, stop_words=None,\n",
       "        ...         use_idf=True)), ('classifier', MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True))])"
      ]
     },
     "execution_count": 170,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Same thing done in pipelines\n",
    "\n",
    "pipeline = Pipeline([\n",
    "  ('counts', CountVectorizer()),\n",
    "  ('tf_idf', TfidfTransformer()),\n",
    "  ('classifier', MultinomialNB())\n",
    "])\n",
    "pipeline.fit( df['msg'], df['label'] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['spam', 'ham'], \n",
       "      dtype='|S4')"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipeline.predict(['100 dating service cal;l 09064012103 box334sk38ch', \"Hey ralph, you coming tonight?\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Using pipeline's featureunion\n",
    "# which will concatinate different preprocessing steps together\n",
    "\n",
    "# A feature union is a pipeline itself\n",
    "\n",
    "from sklearn.datasets import load_iris\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "from sklearn.feature_selection import SelectKBest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "iris = load_iris()\n",
    "X = iris.data\n",
    "X -= X.mean(axis=0)\n",
    "y = iris.target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-2.68420713,  0.32660731],\n",
       "       [-2.71539062, -0.16955685],\n",
       "       [-2.88981954, -0.13734561],\n",
       "       [-2.7464372 , -0.31112432],\n",
       "       [-2.72859298,  0.33392456]])"
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "svd = TruncatedSVD(n_components=2, random_state=0)\n",
    "# A singular value decomp (or PCA)\n",
    "svd.fit_transform(X)[:5]\n",
    "# onlt outputs 2 columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-2.35866667],\n",
       "       [-2.35866667],\n",
       "       [-2.45866667],\n",
       "       [-2.25866667],\n",
       "       [-2.35866667]])"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "select = SelectKBest(k=1)\n",
    "# only one column\n",
    "select.fit_transform(X, y)[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-2.68420713,  0.32660731, -2.35866667],\n",
       "       [-2.71539062, -0.16955685, -2.35866667],\n",
       "       [-2.88981954, -0.13734561, -2.45866667],\n",
       "       [-2.7464372 , -0.31112432, -2.25866667],\n",
       "       [-2.72859298,  0.33392456, -2.35866667]])"
      ]
     },
     "execution_count": 154,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fs = FeatureUnion([(\"svd\", svd), (\"select\", select)])\n",
    "fs.fit_transform(X, y)[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.svm import LinearSVC\n",
    "# A crazy pipeline within pipelines\n",
    "# Pipe-ception\n",
    "# I use lienarsvc beacuse I know linear is best kernel (from previous work and linearSVC is then faster)\n",
    "\n",
    "pipeline = Pipeline([\n",
    "    ('feats', FeatureUnion([\n",
    "        ('count', CountVectorizer()), # can pass in either a pipeline\n",
    "        ('tfidf', TfidfVectorizer()) # or a transformer\n",
    "    ])),\n",
    "    ('clf', LinearSVC())  # classifier\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.98438608496832014"
      ]
     },
     "execution_count": 156,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cross_val_score(pipeline, df['msg'], df['label'], cv=5, scoring='accuracy').mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('feats', FeatureUnion(n_jobs=1,\n",
       "       transformer_list=[('count', CountVectorizer(analyzer=u'word', binary=False, decode_error=u'strict',\n",
       "        dtype=<type 'numpy.int64'>, encoding=u'utf-8', input=u'content',\n",
       "        lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
       "        ngram_r...ax_iter=1000,\n",
       "     multi_class='ovr', penalty='l2', random_state=None, tol=0.0001,\n",
       "     verbose=0))])"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipeline.fit(df['msg'], df['label'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['spam', 'ham'], dtype=object)"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipeline.predict(['100 dating service cal;l 09064012103 box334sk38ch', \"Hey ralph, you coming tonight?\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Exercise\n",
    "# 1. Make  pipeline that takes in a union of three PCA components and a tfidf vectorizer of up to 2 ngrams \n",
    "# coupled with a linearSVC\n",
    "\n",
    "# 2. Do a grid search to find the best params\n",
    "\n",
    "# 3. fit the entire pipeline with the entire data set\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}