kaggle digits.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "load packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from sklearn import svm, model_selection\n",
    "from sklearn.metrics import classification_report, confusion_matrix\n",
    "from sklearn.externals import joblib\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "load and split kaggle training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('data/train.csv')\n",
    "#arr = df.values\n",
    "\n",
    "X = df.loc[:, df.columns != 'label'].as_matrix().astype('uint8')\n",
    "Y = df.label.as_matrix().tolist()\n",
    "\n",
    "X_train, X_test, Y_train, Y_test = model_selection.train_test_split(X, Y, test_size=0.25)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "visualize first entry"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAADYVJREFUeJzt3V2MVPUZx/HfU6teABKVwaJI1xJj\naoyCjpsmarP4Fmww+IJGNLhNtOuFJjXxoupNvfEljVXRVM2qyJKgQmxF4lvxpcSaqHFRFJS2itnq\nyrosoaFyVdGnF3toV9z5zzBzZs4sz/eTkJk5zzlznoz+9szM/8z5m7sLQDw/KLoBAMUg/EBQhB8I\nivADQRF+ICjCDwRF+IGgCD8QFOEHgvphK3c2bdo07+joaOUugVAGBga0Y8cOq2XdhsJvZvMlLZV0\nkKRH3f2u1PodHR3q7+9vZJcAEsrlcs3r1v2238wOkvQHSRdIOlHSYjM7sd7nA9BajXzm75T0ibt/\n6u7/kfSUpIX5tAWg2RoJ/zGSPh/zeDBb9h1m1mNm/WbWPzIy0sDuAOSpkfCP96XC934f7O697l52\n93KpVGpgdwDy1Ej4ByUdO+bxTEnbGmsHQKs0Ev53JB1vZseZ2SGSrpC0Np+2ADRb3UN97r7HzG6Q\n9GeNDvUtc/cPc+sMQFM1NM7v7i9IeiGnXgC0EKf3AkERfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8I\nivADQRF+ICjCDwRF+IGgCD8QVEsv3R3Vjh07kvXTTz89WV+1alWy3tnZud89ARz5gaAIPxAU4QeC\nIvxAUIQfCIrwA0ERfiAoxvlbYNOmTcn6F198kawvWrQoWf/oo48q1iZPnpzcFnFx5AeCIvxAUIQf\nCIrwA0ERfiAowg8ERfiBoBoa5zezAUlfSfpG0h53L+fR1IFm3rx5yfrUqVOT9cHBwWR9/fr1FWsL\nFixIbou48jjJZ567p69WAaDt8LYfCKrR8LukdWa2wcx68mgIQGs0+rb/DHffZmbTJb1sZn9z99fH\nrpD9UeiRpFmzZjW4OwB5aejI7+7bstvtkp6R9L0rSbp7r7uX3b1cKpUa2R2AHNUdfjObZGZT9t6X\ndL6kzXk1BqC5Gnnbf5SkZ8xs7/M84e4v5dIVgKarO/zu/qmkU3LsBUALMdQHBEX4gaAIPxAU4QeC\nIvxAUIQfCIrwA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QFOEH\ngiL8QFCEHwgqj1l60WSHHnposn700Ue3qBMcSDjyA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQVcf5\nzWyZpAWStrv7SdmyIyStktQhaUDS5e7+r+a1GduUKVOS9VNPPbVFneBAUsuRf7mk+fssu1nSq+5+\nvKRXs8cAJpCq4Xf31yXt3GfxQkl92f0+SRfl3BeAJqv3M/9R7j4kSdnt9PxaAtAKTf/Cz8x6zKzf\nzPpHRkaavTsANao3/MNmNkOSstvtlVZ09153L7t7uVQq1bk7AHmrN/xrJXVn97slPZtPOwBapWr4\nzexJSW9KOsHMBs3sGkl3STrPzD6WdF72GMAEUnWc390XVyidk3MvYU2aNClZ37ZtW7K+fv36irWu\nrq46Oqrdzp37DgR91wMPPFCxtmvXruS2V111VbJ+2mmnJetI4ww/ICjCDwRF+IGgCD8QFOEHgiL8\nQFBcursNXHLJJcn6fffdl6zv3r07z3a+Y+vWrcn6WWedlax/+eWXde975cqVyfpLL72UrM+dO7fu\nfUfAkR8IivADQRF+ICjCDwRF+IGgCD8QFOEHgmKcfwJw92S9kbH0aj/JrTaOPzQ0lKynfq589dVX\nJ7fdsGFDsn7uuecm6y+++GLFWmdnZ3LbCDjyA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQjPO3gcMO\nOyxZN7Nk/ZVXXqlYu/baa5Pbpi6tLTV2DoEk9fX1VaxdeumlyW3ff//9ZP3MM89M1h999NGKNcb5\nOfIDYRF+ICjCDwRF+IGgCD8QFOEHgiL8QFBVx/nNbJmkBZK2u/tJ2bLbJP1K0ki22q3u/kKzmjzQ\nVbtu/+233173c1cbp3/44Yfrfm5J6u7uTtYvvPDCup/7lFNOSdarvW5r1qypWOvt7a2rpwNJLUf+\n5ZLmj7P8Xnefk/0j+MAEUzX87v66pPTlXgBMOI185r/BzD4ws2VmdnhuHQFoiXrD/5Ck2ZLmSBqS\n9PtKK5pZj5n1m1n/yMhIpdUAtFhd4Xf3YXf/xt2/lfSIpIq/knD3Xncvu3u5VCrV2yeAnNUVfjOb\nMebhxZI259MOgFapZajvSUldkqaZ2aCk30rqMrM5klzSgKTrmtgjgCaoGn53XzzO4sea0EtYJ598\ncrI+f/54I63/9+abb1aspX5PL0nDw8PJejXLly9vaPtGHHnkkYXt+0DAGX5AUIQfCIrwA0ERfiAo\nwg8ERfiBoLh09wRw9tlnJ+vPP/98xdott9zS0L7vvvvuhrZvptWrVxfdwoTGkR8IivADQRF+ICjC\nDwRF+IGgCD8QFOEHgmKcfwK47LLLkvUHH3ywYm3r1q0N7Xvu3LkNbd+IapcVr3ZZuKlTp+bZzgGH\nIz8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBMU4/wQwc+bMZP3OO++sWLvyyiuT2+7ZsydZf+2115L1\nefPmJeuN7HvFihXJ+tdff52sd3R07G9LoXDkB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgqo7zm9mx\nklZI+pGkbyX1uvtSMztC0ipJHZIGJF3u7v9qXquoZNGiRRVrGzduTG57xx13JOtPP/10sn7CCSck\n67Nnz65YW7p0aXLbt956K1mvdq2B5557LlmPrpYj/x5JN7n7TyX9TNL1ZnaipJslverux0t6NXsM\nYIKoGn53H3L3d7P7X0naIukYSQsl9WWr9Um6qFlNAsjffn3mN7MOSXMlvS3pKHcfkkb/QEianndz\nAJqn5vCb2WRJf5R0o7v/ez+26zGzfjPrr3bNNQCtU1P4zexgjQZ/pbv/KVs8bGYzsvoMSdvH29bd\ne9297O7lUqmUR88AclA1/GZmkh6TtMXd7xlTWiupO7vfLenZ/NsD0Cy1/KT3DElLJG0ys73jRrdK\nukvSajO7RtJnktLXl0Yhenp6kvXNmzcn6+vWrUvWlyxZkqyPHjuao9q+p0/na6iUquF39zckVfov\neE6+7QBoFc7wA4Ii/EBQhB8IivADQRF+ICjCDwTFpbsPcLNmzUrW16xZk6y/9957yfr999+frO/a\ntavufXd1dSXrCxYsSNaRxpEfCIrwA0ERfiAowg8ERfiBoAg/EBThB4JinB9J1S6P/fjjj7eoE+SN\nIz8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8EVTX8\nZnasmf3FzLaY2Ydm9uts+W1m9oWZbcz+/aL57QLISy0X89gj6SZ3f9fMpkjaYGYvZ7V73f3u5rUH\noFmqht/dhyQNZfe/MrMtko5pdmMAmmu/PvObWYekuZLezhbdYGYfmNkyMzu8wjY9ZtZvZv0jIyMN\nNQsgPzWH38wmS/qjpBvd/d+SHpI0W9Icjb4z+P1427l7r7uX3b1cKpVyaBlAHmoKv5kdrNHgr3T3\nP0mSuw+7+zfu/q2kRyR1Nq9NAHmr5dt+k/SYpC3ufs+Y5TPGrHaxpM35twegWWr5tv8MSUskbTKz\njdmyWyUtNrM5klzSgKTrmtIhgKao5dv+NyTZOKUX8m8HQKtwhh8QFOEHgiL8QFCEHwiK8ANBEX4g\nKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiAoc/fW7cxsRNI/xyyaJmlHyxrYP+3aW7v2JdFb\nvfLs7cfuXtP18loa/u/t3Kzf3cuFNZDQrr21a18SvdWrqN542w8ERfiBoIoOf2/B+09p197atS+J\n3upVSG+FfuYHUJyij/wAClJI+M1svpn93cw+MbObi+ihEjMbMLNN2czD/QX3sszMtpvZ5jHLjjCz\nl83s4+x23GnSCuqtLWZuTswsXehr124zXrf8bb+ZHSTpH5LOkzQo6R1Ji939o5Y2UoGZDUgqu3vh\nY8Jm9nNJuyWtcPeTsmW/k7TT3e/K/nAe7u6/aZPebpO0u+iZm7MJZWaMnVla0kWSfqkCX7tEX5er\ngNetiCN/p6RP3P1Td/+PpKckLSygj7bn7q9L2rnP4oWS+rL7fRr9n6flKvTWFtx9yN3fze5/JWnv\nzNKFvnaJvgpRRPiPkfT5mMeDaq8pv13SOjPbYGY9RTczjqOyadP3Tp8+veB+9lV15uZW2mdm6bZ5\n7eqZ8TpvRYR/vNl/2mnI4Qx3P1XSBZKuz97eojY1zdzcKuPMLN0W6p3xOm9FhH9Q0rFjHs+UtK2A\nPsbl7tuy2+2SnlH7zT48vHeS1Ox2e8H9/E87zdw83szSaoPXrp1mvC4i/O9IOt7MjjOzQyRdIWlt\nAX18j5lNyr6IkZlNknS+2m/24bWSurP73ZKeLbCX72iXmZsrzSytgl+7dpvxupCTfLKhjPskHSRp\nmbvf3vImxmFmP9Ho0V4ancT0iSJ7M7MnJXVp9Fdfw5J+K2mNpNWSZkn6TNJl7t7yL94q9Nal0beu\n/5u5ee9n7Bb3dqakv0raJOnbbPGtGv18Xdhrl+hrsQp43TjDDwiKM/yAoAg/EBThB4Ii/EBQhB8I\nivADQRF+ICjCDwT1X43ez+fLaXN4AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1c4a7b4ef60>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "6"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "plt.imshow(X_train[0].reshape(28,28), cmap=plt.cm.gray_r)\n",
    "plt.show()\n",
    "\n",
    "Y_train[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "train a linear support vector classifier and predict the labels of the test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = svm.LinearSVC()\n",
    "clf.fit(X_train, Y_train)\n",
    "\n",
    "predicted = clf.predict(X_test)\n",
    "#sorted(clf.cv_results_.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification report for classifier LinearSVC(C=1.0, class_weight=None, dual=True, fit_intercept=True,\n",
      "     intercept_scaling=1, loss='squared_hinge', max_iter=1000,\n",
      "     multi_class='ovr', penalty='l2', random_state=None, tol=0.0001,\n",
      "     verbose=0):\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.92      0.97      0.94      1039\n",
      "          1       0.93      0.93      0.93      1187\n",
      "          2       0.90      0.78      0.84      1037\n",
      "          3       0.83      0.87      0.85      1088\n",
      "          4       0.64      0.95      0.76      1021\n",
      "          5       0.93      0.60      0.73       951\n",
      "          6       0.91      0.93      0.92      1063\n",
      "          7       0.89      0.89      0.89      1106\n",
      "          8       0.70      0.87      0.78       963\n",
      "          9       0.88      0.55      0.68      1045\n",
      "\n",
      "avg / total       0.86      0.84      0.84     10500\n",
      "\n",
      "\n",
      "Confusion matrix:\n",
      "[[1004    2    1    7    4    2    9    2    7    1]\n",
      " [   1 1108    9    1    5    2    3    2   54    2]\n",
      " [  17   29  811   30   37    1   17   10   83    2]\n",
      " [   7    5   31  950   11    9    2   10   54    9]\n",
      " [   2    4   11    2  974    1    9    4    5    9]\n",
      " [  18    8    9   93   82  571   45   11  102   12]\n",
      " [  12    4    8    1   34    8  986    1    9    0]\n",
      " [   3    4   13   14   41    1    1  986    9   34]\n",
      " [  14   19    3   27   27   12   10    3  842    6]\n",
      " [  10    8    3   16  312    5    2   76   38  575]]\n"
     ]
    }
   ],
   "source": [
    "print(\"Classification report for classifier %s:\\n%s\\n\" % (clf, classification_report(Y_test, predicted)))\n",
    "print(\"Confusion matrix:\\n%s\" % confusion_matrix(Y_test, predicted))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Score: 0.838761904762\n"
     ]
    }
   ],
   "source": [
    "#print(svc.coef_)\n",
    "#print(svc.intercept_)\n",
    "print(\"Score: %s\" % clf.score(X_test, Y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "save the model for later use"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['mnist.model.sav']"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "joblib.dump(clf, 'mnist.model.sav')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "retrain the model by tuning the parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.90546031746\n",
      "1e-06\n",
      "Score: 0.908285714286\n"
     ]
    }
   ],
   "source": [
    "Cs = np.logspace(-6, -1, 10)\n",
    "clf = model_selection.GridSearchCV(estimator=svm.LinearSVC(), param_grid=dict(C=Cs), n_jobs=-1)\n",
    "clf.fit(X_train, Y_train)\n",
    "print(clf.best_score_)\n",
    "print(clf.best_estimator_.C)\n",
    "\n",
    "print(\"Score: %s\" % clf.score(X_test, Y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification report for classifier GridSearchCV(cv=None, error_score='raise',\n",
      "       estimator=LinearSVC(C=1.0, class_weight=None, dual=True, fit_intercept=True,\n",
      "     intercept_scaling=1, loss='squared_hinge', max_iter=1000,\n",
      "     multi_class='ovr', penalty='l2', random_state=None, tol=0.0001,\n",
      "     verbose=0),\n",
      "       fit_params=None, iid=True, n_jobs=-1,\n",
      "       param_grid={'C': array([  1.00000e-06,   3.59381e-06,   1.29155e-05,   4.64159e-05,\n",
      "         1.66810e-04,   5.99484e-04,   2.15443e-03,   7.74264e-03,\n",
      "         2.78256e-02,   1.00000e-01])},\n",
      "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
      "       scoring=None, verbose=0):\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.94      0.97      0.96      1039\n",
      "          1       0.94      0.97      0.95      1187\n",
      "          2       0.92      0.88      0.90      1037\n",
      "          3       0.89      0.89      0.89      1088\n",
      "          4       0.93      0.92      0.92      1021\n",
      "          5       0.90      0.82      0.86       951\n",
      "          6       0.94      0.96      0.95      1063\n",
      "          7       0.92      0.92      0.92      1106\n",
      "          8       0.84      0.86      0.85       963\n",
      "          9       0.86      0.88      0.87      1045\n",
      "\n",
      "avg / total       0.91      0.91      0.91     10500\n",
      "\n",
      "\n",
      "Confusion matrix:\n",
      "[[1010    0    1    3    2    2    9    0   10    2]\n",
      " [   0 1152    8    0    2    2    3    2   15    3]\n",
      " [  11   18  912   10    7    5   13   18   41    2]\n",
      " [   5    6   29  964    3   26    4   11   28   12]\n",
      " [   2    3   10    2  936    4    5    2   10   47]\n",
      " [  12   11    2   48   12  780   23    7   40   16]\n",
      " [   9    3    6    3    7   12 1018    0    4    1]\n",
      " [   5    5   14    5    6    3    3 1015    3   47]\n",
      " [   7   24    5   29    6   29    7    5  831   20]\n",
      " [  12    4    4   19   30    5    0   40   12  919]]\n"
     ]
    }
   ],
   "source": [
    "predicted = clf.predict(X_test)\n",
    "\n",
    "print(\"Classification report for classifier %s:\\n%s\\n\" % (clf, classification_report(Y_test, predicted)))\n",
    "print(\"Confusion matrix:\\n%s\" % confusion_matrix(Y_test, predicted))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "save the tuned model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['mnist.tunedmodel.sav']"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "joblib.dump(clf, 'mnist.tunedmodel.sav')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "train a model using random forest classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'max_features': 'sqrt', 'n_estimators': 700}\n",
      "Score: 0.96780952381\n"
     ]
    }
   ],
   "source": [
    "rfc = RandomForestClassifier(n_estimators=50, n_jobs=-1, max_features='sqrt', oob_score=True)\n",
    "param_grid = { \n",
    "    'n_estimators': [200, 700],\n",
    "    'max_features': ['auto', 'sqrt', 'log2']\n",
    "}\n",
    "clf = model_selection.GridSearchCV(estimator=rfc, param_grid=param_grid, cv= 5)\n",
    "clf.fit(X_train, Y_train)\n",
    "print(clf.best_params_)\n",
    "\n",
    "clf.fit(X_train,Y_train)\n",
    "\n",
    "print(\"Score: %s\" % clf.score(X_test, Y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification report for classifier GridSearchCV(cv=5, error_score='raise',\n",
      "       estimator=RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
      "            max_depth=None, max_features='sqrt', max_leaf_nodes=None,\n",
      "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
      "            min_samples_leaf=1, min_samples_split=2,\n",
      "            min_weight_fraction_leaf=0.0, n_estimators=50, n_jobs=-1,\n",
      "            oob_score=True, random_state=None, verbose=0, warm_start=False),\n",
      "       fit_params=None, iid=True, n_jobs=1,\n",
      "       param_grid={'n_estimators': [200, 700], 'max_features': ['auto', 'sqrt', 'log2']},\n",
      "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
      "       scoring=None, verbose=0):\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.98      0.99      0.98      1039\n",
      "          1       0.98      0.98      0.98      1187\n",
      "          2       0.95      0.97      0.96      1037\n",
      "          3       0.96      0.95      0.95      1088\n",
      "          4       0.97      0.98      0.97      1021\n",
      "          5       0.98      0.96      0.97       951\n",
      "          6       0.98      0.99      0.98      1063\n",
      "          7       0.98      0.96      0.97      1106\n",
      "          8       0.96      0.96      0.96       963\n",
      "          9       0.95      0.95      0.95      1045\n",
      "\n",
      "avg / total       0.97      0.97      0.97     10500\n",
      "\n",
      "\n",
      "Confusion matrix:\n",
      "[[1026    0    1    0    1    1    3    0    7    0]\n",
      " [   0 1168    8    2    2    1    2    2    2    0]\n",
      " [   4    5 1002    3    6    0    4    5    8    0]\n",
      " [   3    2   17 1030    0    8    3    7   11    7]\n",
      " [   1    0    1    1  997    0    3    1    3   14]\n",
      " [   3    1    1   12    1  917    9    1    2    4]\n",
      " [   2    2    1    0    1    4 1052    0    1    0]\n",
      " [   1    4   19    2    7    0    0 1060    2   11]\n",
      " [   2    4    3   10    3    3    2    1  922   13]\n",
      " [   9    2    0   18   12    1    0   10    5  988]]\n"
     ]
    }
   ],
   "source": [
    "predicted = clf.predict(X_test)\n",
    "\n",
    "print(\"Classification report for classifier %s:\\n%s\\n\" % (clf, classification_report(Y_test, predicted)))\n",
    "print(\"Confusion matrix:\\n%s\" % confusion_matrix(Y_test, predicted))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['mnist.randomforestmodel.sav']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "joblib.dump(clf, 'mnist.randomforestmodel.sav')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "load the trained model and the kaggle sample set then visualize the first entry"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf2 = joblib.load('mnist.randomforestmodel.sav')\n",
    "\n",
    "df = pd.read_csv('data/test.csv')\n",
    "X_sample = df.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAADkdJREFUeJzt3X+sVPWZx/HPs3fxB7YihguLVvZW\nQjYa4oKZkE1oVkyVWFOD/aOkhGwQdS9/VLNNmijxH/zDTcxmaRfjTQ1dCJC00JqCopLdqlkE4oYw\nGCxUdq0xF3r3IlwCUdBEIjz7xz3XXODOd4aZc+YM93m/EjIz55kz3ycTPvfMzHfOfM3dBSCevyi7\nAQDlIPxAUIQfCIrwA0ERfiAowg8ERfiBoAg/EBThB4L6y3YONmXKFO/p6WnnkEAo/f39OnnypDVy\n35bCb2YPSFojqUvSv7v786n79/T0qFqttjIkgIRKpdLwfZt+2W9mXZL6JH1P0p2SlpjZnc0+HoD2\nauU9/zxJH7n7x+5+TtIWSYvyaQtA0VoJ/62S/jzq9kC27SJm1mtmVTOrDg0NtTAcgDy1Ev6xPlS4\n7Pxgd1/r7hV3r3R3d7cwHIA8tRL+AUm3jbr9LUmDrbUDoF1aCf8+SbPM7Ntmdo2kH0nank9bAIrW\n9FSfu39lZk9I+k8NT/Wtd/c/5tYZgEK1NM/v7jsk7cipFwBtxNd7gaAIPxAU4QeCIvxAUIQfCIrw\nA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiq\nrUt0ozlffPFFsv7ll1+2qZPL7dy5M1lft25d04+9Zs2aZH3mzJlNPzY48gNhEX4gKMIPBEX4gaAI\nPxAU4QeCIvxAUC3N85tZv6Qzks5L+srdK3k0hYutWrUqWV+9enWbOmmv5557ruwWxrU8vuRzr7uf\nzOFxALQRL/uBoFoNv0v6vZntN7PePBoC0B6tvuyf7+6DZjZV0ptm9j/uvmv0HbI/Cr2SNGPGjBaH\nA5CXlo787j6YXZ6QtE3SvDHus9bdK+5e6e7ubmU4ADlqOvxmdoOZfXPkuqSFkg7l1RiAYrXysn+a\npG1mNvI4v3b3/8ilKwCFazr87v6xpL/NsZew9uzZk6xv3ry5TZ10luXLlyfr119/fbL+0ksv1azd\nddddTfU0njDVBwRF+IGgCD8QFOEHgiL8QFCEHwiKn+7uAL296dMiBgcH29RJZ3n//fdb2n/RokU1\nay+//HJy30pl/J+dzpEfCIrwA0ERfiAowg8ERfiBoAg/EBThB4Jinr8DvPjii8n60qVLk/Xjx4/n\n2c5FXnjhhWT9vvvua/qx33jjjWS93k+W11u6/MiRIzVrW7duTe47d+7cZL2rqytZvxpw5AeCIvxA\nUIQfCIrwA0ERfiAowg8ERfiBoMzd2zZYpVLxarXatvHGi3feeSdZ379/f2FjP/TQQ8n6rFmzChv7\n7rvvTtYPHDhQ2NinT59O1idNmlTY2K2oVCqqVqvWyH058gNBEX4gKMIPBEX4gaAIPxAU4QeCIvxA\nUHXP5zez9ZK+L+mEu8/Ott0s6TeSeiT1S1rs7umJUTTtnnvuaal+terr60vW58+f36ZOxqdGjvwb\nJD1wybaVkt5291mS3s5uA7iK1A2/u++SdOqSzYskbcyub5T0cM59AShYs+/5p7n7MUnKLqfm1xKA\ndij8Az8z6zWzqplVh4aGih4OQIOaDf9xM5suSdnliVp3dPe17l5x90p3d3eTwwHIW7Ph3y5pWXZ9\nmaRX82kHQLvUDb+ZbZb035L+xswGzOwxSc9Lut/M/iTp/uw2gKtI3Xl+d19So/TdnHsBLtKp58yP\nF3zDDwiK8ANBEX4gKMIPBEX4gaAIPxAUS3SjY+3bt6/sFsY1jvxAUIQfCIrwA0ERfiAowg8ERfiB\noAg/EBTz/OhYa9asKbuFcY0jPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8ExTz/OLd79+5k/cMPP0zW\nu7q6kvVHHnnkSlv62sGDB5P1U6cuXR82P/WW954wYUJhY3cKjvxAUIQfCIrwA0ERfiAowg8ERfiB\noAg/EFTdeX4zWy/p+5JOuPvsbNuzkv5R0lB2t2fcfUdRTXaCzz//vGbts88+S+77yiuvJOtTp05N\n1vv6+pL1lHrz+IODg8l6vXn+TZs2XXFPIwYGBpL1o0ePNv3YkjR79uyatS1btiT3nThxYktjXw0a\nOfJvkPTAGNt/7u5zsn/jOvjAeFQ3/O6+S1JxX7UCUIpW3vM/YWZ/MLP1ZjY5t44AtEWz4f+FpJmS\n5kg6Jml1rTuaWa+ZVc2sOjQ0VOtuANqsqfC7+3F3P+/uFyT9UtK8xH3XunvF3Svd3d3N9gkgZ02F\n38ymj7r5A0mH8mkHQLs0MtW3WdICSVPMbEDSKkkLzGyOJJfUL2lFgT0CKEDd8Lv7kjE2ryugl0J9\n8MEHyfqOHenZynfffbdmrd48/tXs/PnzyfrOnTvb00gTzp49W7O2efPm5L5PPvlksn7ttdc21VMn\n4Rt+QFCEHwiK8ANBEX4gKMIPBEX4gaDC/HT366+/nqyvXLmyTZ1c7rrrrkvWb7/99mQ9dbrxkSNH\nmuppPOjv769Ze+qpp5L7HjqU/t5aveXDJ02alKx3Ao78QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxBU\nmHn+p59+Olk3s8LGXrBgQbK+dOnSZP2xxx5L1lPz2YsXL07uW61Wk/VW3XjjjTVr9eba63nrrbeS\n9VZON673k+T1fq5969atTY/dLhz5gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiAoc/e2DVapVLzoeeVa\n6s3jFznPX+/c7ptuuqmwsU+dSq+xeubMmZYef9q0acn6xo0ba9YWLlzY0tinT59O1h999NGatb17\n9yb3/eSTT5rqacSFCxda2r9ZlUpF1Wq1of/MHPmBoAg/EBThB4Ii/EBQhB8IivADQRF+IKi65/Ob\n2W2SNkn6K0kXJK119zVmdrOk30jqkdQvabG7pydeS7R8+fJkfcOGDYWN/emnn7ZUL9KcOXOS9ccf\nfzxZv+OOO5L1e++994p7atTkyZOT9W3bttWs7dq1K7nvgw8+mKzX+52Eq0EjR/6vJP3U3e+Q9HeS\nfmxmd0paKeltd58l6e3sNoCrRN3wu/sxd38vu35G0mFJt0paJGnk61sbJT1cVJMA8ndF7/nNrEfS\nXEl7JU1z92PS8B8ISVPzbg5AcRoOv5l9Q9LvJP3E3dM/YHbxfr1mVjWz6tDQUDM9AihAQ+E3swka\nDv6v3H3klwmPm9n0rD5d0omx9nX3te5ecfdKd3d3Hj0DyEHd8Nvw6W7rJB1295+NKm2XtCy7vkzS\nq/m3B6AodU/pNbPvSNot6aCGp/ok6RkNv+//raQZko5K+qG7J88fLfOU3nPnziXrJ0+eTNZXrFiR\nZzu56uvrq1mrdzrxhAkTkvWJEyc21dPVrt5Pc9dbVv2aa67Js52GXckpvXXn+d19j6RaD/bdK2kM\nQOfgG35AUIQfCIrwA0ERfiAowg8ERfiBoMIs0V1v3vWWW25J1l977bU820GHSy0tPl5w5AeCIvxA\nUIQfCIrwA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QFOEHgiL8\nQFCEHwiK8ANBEX4gKMIPBEX4gaDqht/MbjOz/zKzw2b2RzP7p2z7s2b2f2Z2IPv3YPHtAshLI4t2\nfCXpp+7+npl9U9J+M3szq/3c3f+1uPYAFKVu+N39mKRj2fUzZnZY0q1FNwagWFf0nt/MeiTNlbQ3\n2/SEmf3BzNab2eQa+/SaWdXMqkNDQy01CyA/DYffzL4h6XeSfuLun0n6haSZkuZo+JXB6rH2c/e1\n7l5x90p3d3cOLQPIQ0PhN7MJGg7+r9x9qyS5+3F3P+/uFyT9UtK84toEkLdGPu03SeskHXb3n43a\nPn3U3X4g6VD+7QEoSiOf9s+X9A+SDprZgWzbM5KWmNkcSS6pX9KKQjoEUIhGPu3fI8nGKO3Ivx0A\n7cI3/ICgCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0GZu7dv\nMLMhSUdGbZoi6WTbGrgyndpbp/Yl0Vuz8uztr929od/La2v4LxvcrOruldIaSOjU3jq1L4nemlVW\nb7zsB4Ii/EBQZYd/bcnjp3Rqb53al0RvzSqlt1Lf8wMoT9lHfgAlKSX8ZvaAmf2vmX1kZivL6KEW\nM+s3s4PZysPVkntZb2YnzOzQqG03m9mbZvan7HLMZdJK6q0jVm5OrCxd6nPXaStet/1lv5l1SfpQ\n0v2SBiTtk7TE3T9oayM1mFm/pIq7lz4nbGZ/L+mspE3uPjvb9i+STrn789kfzsnu/nSH9PaspLNl\nr9ycLSgzffTK0pIelvSISnzuEn0tVgnPWxlH/nmSPnL3j939nKQtkhaV0EfHc/ddkk5dsnmRpI3Z\n9Y0a/s/TdjV66wjufszd38uun5E0srJ0qc9doq9SlBH+WyX9edTtAXXWkt8u6fdmtt/MestuZgzT\nsmXTR5ZPn1pyP5equ3JzO12ysnTHPHfNrHidtzLCP9bqP5005TDf3e+W9D1JP85e3qIxDa3c3C5j\nrCzdEZpd8TpvZYR/QNJto25/S9JgCX2Myd0Hs8sTkrap81YfPj6ySGp2eaLkfr7WSSs3j7WytDrg\nueukFa/LCP8+SbPM7Ntmdo2kH0naXkIflzGzG7IPYmRmN0haqM5bfXi7pGXZ9WWSXi2xl4t0ysrN\ntVaWVsnPXaeteF3Kl3yyqYx/k9Qlab27/3PbmxiDmd2u4aO9NLyI6a/L7M3MNktaoOGzvo5LWiXp\nFUm/lTRD0lFJP3T3tn/wVqO3BRp+6fr1ys0j77Hb3Nt3JO2WdFDShWzzMxp+f13ac5foa4lKeN74\nhh8QFN/wA4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgCD8Q1P8DqkoYlPwbXhwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1c4800e4d68>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(X_sample[0].reshape(28,28), cmap=plt.cm.gray_r)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "predict the labels for the kaggle sample set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2, 0, 9, ..., 3, 9, 2])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y_sample = clf2.predict(X_sample)\n",
    "Y_sample"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "structure kaggle submission according to requirements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "index = [i for i in range(1, len(Y_sample)+1)]\n",
    "index\n",
    "df = pd.DataFrame(Y_sample, index=index, columns=[\"Label\"])\n",
    "df.index.name = \"ImageId\"\n",
    "\n",
    "df.to_csv('data/result.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}