digits_classification.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.metrics import accuracy_score\n",
    "from matplotlib import pyplot as plt\n",
    "%matplotlib inline\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append(\"../..\")\n",
    "\n",
    "import matplotlib_utils\n",
    "from importlib import reload\n",
    "reload(matplotlib_utils)\n",
    "\n",
    "import keras_utils\n",
    "from keras_utils import reset_tf_session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import preprocessed_mnist\n",
    "X_train, y_train, X_val, y_val, X_test, y_test = preprocessed_mnist.load_dataset_from_file()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train [shape (50000, 28, 28)] sample patch:\n",
      " [[ 0.          0.29803922  0.96470588  0.98823529  0.43921569]\n",
      " [ 0.          0.33333333  0.98823529  0.90196078  0.09803922]\n",
      " [ 0.          0.33333333  0.98823529  0.8745098   0.        ]\n",
      " [ 0.          0.33333333  0.98823529  0.56862745  0.        ]\n",
      " [ 0.          0.3372549   0.99215686  0.88235294  0.        ]]\n",
      "A closeup of a sample patch:\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPgAAAD8CAYAAABaQGkdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAACTFJREFUeJzt3U9onAUexvHnMVup0AUPnUNpyqYHEYqwCqFIeysIVYte\nFRQPQi8rVBBEPQhePHgQL16K/xYURdCDFBcpWBHBVUdbxdoKRVysCJ1FxIoSqT4eMoeuNJ03mffN\nm/nt9wOBTDJMHkq+fWfeDDNOIgA1XdH3AADdIXCgMAIHCiNwoDACBwojcKAwAgcKI3CgMAIHCvtL\nFze6devWLCwsdHHTrfv555/7nrAqp0+f7nvCqszSMyV37tzZ94TGRqORzp8/70nX6yTwhYUFDYfD\nLm66dcePH+97wqrs2bOn7wmrsrS01PeExh5//PG+JzT2yCOPNLoed9GBwggcKIzAgcIIHCiMwIHC\nCBwojMCBwggcKIzAgcIIHCiMwIHCCBwojMCBwggcKIzAgcIIHCisUeC299v+0vYZ2w91PQpAOyYG\nbntO0tOSbpa0S9Kdtnd1PQzA9JocwXdLOpPkqyS/SnpF0u3dzgLQhiaBb5f0zUWXz46/BmCDa+0k\nm+2Dtoe2h6PRqK2bBTCFJoF/K2nHRZfnx1/7H0kOJ1lMsjgYDNraB2AKTQL/SNI1tnfavlLSHZLe\n6HYWgDZMfF30JBds3yfpLUlzkp5LcrLzZQCm1uiND5K8KenNjrcAaBnPZAMKI3CgMAIHCiNwoDAC\nBwojcKAwAgcKI3CgMAIHCiNwoDACBwojcKAwAgcKI3CgMAIHCiNwoDACBwpr9Ioulf3yyy99T1iV\npaWlviesyrZt2/qe0NiBAwf6ntDYE0880eh6HMGBwggcKIzAgcIIHCiMwIHCCBwojMCBwggcKIzA\ngcIIHCiMwIHCCBwojMCBwggcKIzAgcIIHCiMwIHCJgZu+znb52x/vh6DALSnyRH8BUn7O94BoAMT\nA0/yrqTv12ELgJbxGBworLXAbR+0PbQ9HI1Gbd0sgCm0FniSw0kWkywOBoO2bhbAFLiLDhTW5M9k\nL0t6X9K1ts/avrf7WQDaMPGdTZLcuR5DALSPu+hAYQQOFEbgQGEEDhRG4EBhBA4URuBAYQQOFEbg\nQGEEDhRG4EBhBA4URuBAYQQOFEbgQGEEDhQ28QUfgGls3ry57wmNbdmype8JjV1xRbNjM0dwoDAC\nBwojcKAwAgcKI3CgMAIHCiNwoDACBwojcKAwAgcKI3CgMAIHCiNwoDACBwojcKAwAgcKI3CgsImB\n295h+5jtL2yftH1oPYYBmF6Tl2y6IOmBJJ/Y/qukj20fTfJFx9sATGniETzJd0k+GX9+XtIpSdu7\nHgZgeqt6DG57QdINkj7oYgyAdjUO3PYWSa9Juj/Jj5f4/kHbQ9vD0WjU5kYAa9QocNubtBz3S0le\nv9R1khxOsphkcTAYtLkRwBo1OYtuSc9KOpXkye4nAWhLkyP4Xkl3S9pn+8T445aOdwFowcQ/kyV5\nT5LXYQuAlvFMNqAwAgcKI3CgMAIHCiNwoDACBwojcKAwAgcKI3CgMAIHCiNwoDACBwojcKAwAgcK\nI3CgMAIHCiNwoLAmb3wArNk999zT94T/axzBgcIIHCiMwIHCCBwojMCBwggcKIzAgcIIHCiMwIHC\nCBwojMCBwggcKIzAgcIIHCiMwIHCCBwojMCBwiYGbnuz7Q9tf2r7pO3H1mMYgOk1ecmmJUn7kvxk\ne5Ok92z/K8m/O94GYEoTA08SST+NL24af6TLUQDa0egxuO052ycknZN0NMkH3c4C0IZGgSf5Lcn1\nkuYl7bZ93Z+vY/ug7aHt4Wg0ansngDVY1Vn0JD9IOiZp/yW+dzjJYpLFwWDQ1j4AU2hyFn1g++rx\n51dJuknS6a6HAZhek7Po2yT90/aclv9DeDXJkW5nAWhDk7Pon0m6YR22AGgZz2QDCiNwoDACBwoj\ncKAwAgcKI3CgMAIHCiNwoDACBwojcKAwAgcKI3CgMAIHCiNwoDACBwojcKCwJq/oUtryq0LPjlnb\n+/zzz/c9obFHH3207wmt4wgOFEbgQGEEDhRG4EBhBA4URuBAYQQOFEbgQGEEDhRG4EBhBA4URuBA\nYQQOFEbgQGEEDhRG4EBhBA4U1jhw23O2j9s+0uUgAO1ZzRH8kKRTXQ0B0L5Ggduel3SrpGe6nQOg\nTU2P4E9JelDS7x1uAdCyiYHbPiDpXJKPJ1zvoO2h7eFoNGptIIC1a3IE3yvpNttfS3pF0j7bL/75\nSkkOJ1lMsjgYDFqeCWAtJgae5OEk80kWJN0h6e0kd3W+DMDU+Ds4UNiq3tkkyTuS3ulkCYDWcQQH\nCiNwoDACBwojcKAwAgcKI3CgMAIHCiNwoDACBwojcKAwAgcKI3CgMAIHCiNwoDACBwojcKAwAgcK\nc5L2b9QeSfpPyze7VdJ/W77NLs3S3lnaKs3W3q62/i3JxFc37STwLtgeJlnse0dTs7R3lrZKs7W3\n763cRQcKI3CgsFkK/HDfA1ZplvbO0lZptvb2unVmHoMDWL1ZOoIDWKWZCNz2fttf2j5j+6G+91yO\n7edsn7P9ed9bJrG9w/Yx21/YPmn7UN+bVmJ7s+0PbX863vpY35uasD1n+7jtI338/A0fuO05SU9L\nulnSLkl32t7V76rLekHS/r5HNHRB0gNJdkm6UdI/NvC/7ZKkfUn+Lul6Sftt39jzpiYOSTrV1w/f\n8IFL2i3pTJKvkvyq5Xc4vb3nTStK8q6k7/ve0USS75J8Mv78vJZ/Ebf3u+rSsuyn8cVN448NfQLJ\n9rykWyU909eGWQh8u6RvLrp8Vhv0l3CW2V6QdIOkD/pdsrLx3d0Tks5JOppkw24de0rSg5J+72vA\nLASOjtneIuk1Sfcn+bHvPStJ8luS6yXNS9pt+7q+N63E9gFJ55J83OeOWQj8W0k7Lro8P/4aWmB7\nk5bjfinJ633vaSLJD5KOaWOf69gr6TbbX2v5YeU+2y+u94hZCPwjSdfY3mn7Skl3SHqj500l2Lak\nZyWdSvJk33sux/bA9tXjz6+SdJOk0/2uWlmSh5PMJ1nQ8u/s20nuWu8dGz7wJBck3SfpLS2fBHo1\nycl+V63M9suS3pd0re2ztu/te9Nl7JV0t5aPLifGH7f0PWoF2yQds/2Zlv/TP5qklz89zRKeyQYU\ntuGP4ADWjsCBwggcKIzAgcIIHCiMwIHCCBwojMCBwv4APqD4Xdwde0AAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f44edbcd518>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "And the whole sample:\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADpdJREFUeJzt3X2MVGWWx/HfkRl8ASWiLUEHbRZx40tis6mQTYZs2Iwz\nQZ0EiS+BqGEMkQkRdcz4FoxZYzSRdWcQ4mpsFiKss8xsGIz8YdZRshEnGSeW4Iro7upiI3SQLiJk\nHI0ODWf/6OukR7ueKqpu1a3u8/0kna665z59Twp+favuU12PubsAxHNS0Q0AKAbhB4Ii/EBQhB8I\nivADQRF+ICjCDwRF+IGgCD8Q1LfaebCzzz7bu7u723lIIJS+vj4dOnTI6tm3qfCb2TxJqyWNk/Qv\n7v5Yav/u7m6Vy+VmDgkgoVQq1b1vw0/7zWycpH+WdKWkSyQtMrNLGv15ANqrmdf8syV94O573P1P\nkn4paX4+bQFotWbCf56kfcPu78+2/QUzW2pmZTMrVyqVJg4HIE8tv9rv7r3uXnL3UldXV6sPB6BO\nzYS/X9K0Yfe/k20DMAo0E/43JM00s+lmNl7SQklb82kLQKs1PNXn7oNmtlzSSxqa6lvv7rtz6wxA\nSzU1z+/uL0p6MadeALQRb+8FgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIrw\nA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IivADQRF+IKi2LtGNsWffvn3J+urVq6vWVq1alRx7\n1113Jet33nlnsj5t2rRkPTrO/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QVFPz/GbWJ+lTScckDbp7\nKY+m0Dn6+/uT9VmzZiXrR44cqVozs+TYJ554IlnfsGFDsl6pVJL16PJ4k8/fu/uhHH4OgDbiaT8Q\nVLPhd0m/MbM3zWxpHg0BaI9mn/bPcfd+MztH0stm9t/uvn34DtkvhaWSdP755zd5OAB5aerM7+79\n2fcBSc9Lmj3CPr3uXnL3UldXVzOHA5CjhsNvZhPM7PSvbkv6gaR38moMQGs187R/iqTns+mab0n6\nN3f/j1y6AtByDYff3fdIujzHXlCAvXv3Jutz585N1g8fPpysp+byJ02alBx78sknJ+sDAwPJ+p49\ne6rWLrjgguTYcePGJetjAVN9QFCEHwiK8ANBEX4gKMIPBEX4gaD46O4x4OjRo1Vrtaby5s2bl6zX\n+mjuZvT09CTrjz76aLI+Z86cZH3mzJlVa729vcmxS5YsSdbHAs78QFCEHwiK8ANBEX4gKMIPBEX4\ngaAIPxAU8/xjwD333FO19uSTT7axkxPz6quvJuufffZZsr5gwYJkfcuWLVVrO3fuTI6NgDM/EBTh\nB4Ii/EBQhB8IivADQRF+ICjCDwTFPP8oUOtv6p977rmqNXdv6ti15tKvvfbaZP2mm26qWps2bVpy\n7MUXX5ys33fffcn65s2bq9aafVzGAs78QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxCU1ZrvNLP1kn4o\nacDdL8u2TZb0K0ndkvok3eDu6bWaJZVKJS+Xy022PPb09/cn65dfnl4J/ciRIw0f+8Ybb0zW165d\nm6y/++67yfqOHTuq1hYuXJgce9pppyXrtaSW2Z4wYUJy7O7du5P1Wu9RKEqpVFK5XK6+Lvow9Zz5\nn5X09ZUd7pe0zd1nStqW3QcwitQMv7tvl/TJ1zbPl7Qhu71B0jU59wWgxRp9zT/F3Q9ktz+WNCWn\nfgC0SdMX/HzookHVCwdmttTMymZWrlQqzR4OQE4aDf9BM5sqSdn3gWo7unuvu5fcvdTV1dXg4QDk\nrdHwb5W0OLu9WNIL+bQDoF1qht/MNkn6naS/NrP9ZrZE0mOSvm9m70u6IrsPYBSp+ff87r6oSul7\nOfcyZh06dChZX7lyZbJ++HD6LRRTplS/3jp9+vTk2GXLliXr48ePT9Z7enqaqhfl888/T9Yff/zx\nZH3NmjV5tlMI3uEHBEX4gaAIPxAU4QeCIvxAUIQfCIqP7s7B4OBgsn733Xcn66mP3pakSZMmJesv\nvfRS1dqFF16YHHv06NFkPaoPP/yw6BZajjM/EBThB4Ii/EBQhB8IivADQRF+ICjCDwTFPH8OPvro\no2S91jx+La+//nqyftFFFzX8s0899dSGx2J048wPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0Exz5+D\n2267LVmvtQz6ggULkvVm5vEjO378eNXaSSelz3u1/s3GAs78QFCEHwiK8ANBEX4gKMIPBEX4gaAI\nPxBUzXl+M1sv6YeSBtz9smzbQ5JulVTJdlvh7i+2qslOsHPnzqq17du3J8eaWbJ+/fXXN9QT0lJz\n+bX+TUqlUt7tdJx6zvzPSpo3wvZV7t6TfY3p4ANjUc3wu/t2SZ+0oRcAbdTMa/7lZva2ma03szNz\n6whAWzQa/qclzZDUI+mApJ9V29HMlppZ2czKlUql2m4A2qyh8Lv7QXc/5u7HJa2VNDuxb6+7l9y9\n1NXV1WifAHLWUPjNbOqwuwskvZNPOwDapZ6pvk2S5ko628z2S/oHSXPNrEeSS+qT9OMW9gigBWqG\n390XjbB5XQt66WhffPFF1dqXX36ZHHvuuecm61dffXVDPY11g4ODyfqaNWsa/tnXXXddsr5ixYqG\nf/ZowTv8gKAIPxAU4QeCIvxAUIQfCIrwA0Hx0d1tcMoppyTrEydObFMnnaXWVN7TTz+drN97773J\nend3d9XaAw88kBw7fvz4ZH0s4MwPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0Exz98GN998c9EtFKa/\nv79qbeXKlcmxTz31VLJ+yy23JOtr165N1qPjzA8ERfiBoAg/EBThB4Ii/EBQhB8IivADQTHPXyd3\nb6gmSc8++2yy/uCDDzbSUkfYtGlTsn777bdXrR0+fDg59o477kjWV61alawjjTM/EBThB4Ii/EBQ\nhB8IivADQRF+ICjCDwRVc57fzKZJ2ihpiiSX1Ovuq81ssqRfSeqW1CfpBndPT9yOYmbWUE2S9u/f\nn6w//PDDyfqSJUuS9dNPP71qbffu3cmxzzzzTLL+2muvJet9fX3J+owZM6rWFi5cmBxba54fzann\nzD8o6afufomkv5V0m5ldIul+Sdvcfaakbdl9AKNEzfC7+wF335Hd/lTSe5LOkzRf0oZstw2SrmlV\nkwDyd0Kv+c2sW9IsSb+XNMXdD2SljzX0sgDAKFF3+M1soqRfS/qJu/9heM2H3tw+4hvczWypmZXN\nrFypVJpqFkB+6gq/mX1bQ8H/hbtvyTYfNLOpWX2qpIGRxrp7r7uX3L3U1dWVR88AclAz/DZ0KXud\npPfc/efDSlslLc5uL5b0Qv7tAWiVev6k97uSbpa0y8zeyratkPSYpH83syWS9kq6oTUtjn7Hjh1L\n1mtN9a1bty5Znzx5ctXarl27kmObdeWVVybr8+bNq1pbvnx53u3gBNQMv7v/VlK1iezv5dsOgHbh\nHX5AUIQfCIrwA0ERfiAowg8ERfiBoPjo7jpdeumlVWtXXHFFcuwrr7zS1LFr/UlwahnsWs4555xk\nfdmyZcn6aP7Y8eg48wNBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUMzz1+mMM86oWtu8eXNy7MaNG5P1\nVn5E9SOPPJKs33rrrcn6WWedlWc76CCc+YGgCD8QFOEHgiL8QFCEHwiK8ANBEX4gKBtaaas9SqWS\nl8vlth0PiKZUKqlcLqfXjM9w5geCIvxAUIQfCIrwA0ERfiAowg8ERfiBoGqG38ymmdl/mtm7Zrbb\nzO7Mtj9kZv1m9lb2dVXr2wWQl3o+zGNQ0k/dfYeZnS7pTTN7Oautcvd/al17AFqlZvjd/YCkA9nt\nT83sPUnntboxAK11Qq/5zaxb0ixJv882LTezt81svZmdWWXMUjMrm1m5Uqk01SyA/NQdfjObKOnX\nkn7i7n+Q9LSkGZJ6NPTM4GcjjXP3XncvuXupq6srh5YB5KGu8JvZtzUU/F+4+xZJcveD7n7M3Y9L\nWitpduvaBJC3eq72m6R1kt5z958P2z512G4LJL2Tf3sAWqWeq/3flXSzpF1m9la2bYWkRWbWI8kl\n9Un6cUs6BNAS9Vzt/62kkf4++MX82wHQLrzDDwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIQf\nCIrwA0ERfiAowg8ERfiBoAg/EFRbl+g2s4qkvcM2nS3pUNsaODGd2lun9iXRW6Py7O0Cd6/r8/La\nGv5vHNys7O6lwhpI6NTeOrUvid4aVVRvPO0HgiL8QFBFh7+34OOndGpvndqXRG+NKqS3Ql/zAyhO\n0Wd+AAUpJPxmNs/M/sfMPjCz+4vooRoz6zOzXdnKw+WCe1lvZgNm9s6wbZPN7GUzez/7PuIyaQX1\n1hErNydWli70seu0Fa/b/rTfzMZJ+l9J35e0X9Ibkha5+7ttbaQKM+uTVHL3wueEzezvJP1R0kZ3\nvyzb9o+SPnH3x7JfnGe6+30d0ttDkv5Y9MrN2YIyU4evLC3pGkk/UoGPXaKvG1TA41bEmX+2pA/c\nfY+7/0nSLyXNL6CPjufu2yV98rXN8yVtyG5v0NB/nrar0ltHcPcD7r4ju/2ppK9Wli70sUv0VYgi\nwn+epH3D7u9XZy357ZJ+Y2ZvmtnSopsZwZRs2XRJ+ljSlCKbGUHNlZvb6WsrS3fMY9fIitd544Lf\nN81x97+RdKWk27Kntx3Jh16zddJ0TV0rN7fLCCtL/1mRj12jK17nrYjw90uaNuz+d7JtHcHd+7Pv\nA5KeV+etPnzwq0VSs+8DBffzZ520cvNIK0urAx67TlrxuojwvyFppplNN7PxkhZK2lpAH99gZhOy\nCzEyswmSfqDOW314q6TF2e3Fkl4osJe/0CkrN1dbWVoFP3Ydt+K1u7f9S9JVGrri/3+SHiiihyp9\n/ZWk/8q+dhfdm6RNGnoaeFRD10aWSDpL0jZJ70t6RdLkDurtXyXtkvS2hoI2taDe5mjoKf3bkt7K\nvq4q+rFL9FXI48Y7/ICguOAHBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiCo/wfNDnvJ0xlPmwAA\nAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f45410f8c88>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y_train [shape (50000,)] 10 samples:\n",
      " [5 0 4 1 9 2 1 3 1 4]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([5, 0, 4, ..., 8, 4, 8], dtype=uint8)"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"X_train [shape %s] sample patch:\\n\" % (str(X_train.shape)), X_train[1, 15:20, 5:10])\n",
    "print(\"A closeup of a sample patch:\")\n",
    "plt.imshow(X_train[1, 15:20, 5:10], cmap=\"Greys\")\n",
    "plt.show()\n",
    "print(\"And the whole sample:\")\n",
    "plt.imshow(X_train[1], cmap=\"Greys\")\n",
    "plt.show()\n",
    "print(\"y_train [shape %s] 10 samples:\\n\" % (str(y_train.shape)), y_train[:10])\n",
    "y_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(50000, 784)\n",
      "(10000, 784)\n"
     ]
    }
   ],
   "source": [
    "X_train_flat = X_train.reshape((X_train.shape[0], -1))\n",
    "print(X_train_flat.shape)\n",
    "\n",
    "X_val_flat = X_val.reshape((X_val.shape[0], -1))\n",
    "print(X_val_flat.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(50000, 10)\n",
      "[[ 0.  0.  0.  0.  0.  1.  0.  0.  0.  0.]\n",
      " [ 1.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  1.  0.  0.  0.  0.  0.]] [5 0 4]\n"
     ]
    }
   ],
   "source": [
    "import keras\n",
    "\n",
    "y_train_oh = keras.utils.to_categorical(y_train, 10)\n",
    "y_val_oh = keras.utils.to_categorical(y_val, 10)\n",
    "\n",
    "print(y_train_oh.shape)\n",
    "print(y_train_oh[:3], y_train[:3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "'''USING LINEAR MODEL'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "s = reset_tf_session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "W = tf.get_variable(\"W\", shape = (784,10), dtype = tf.float32, trainable = True)\n",
    "b = tf.get_variable(\"b\", shape = (1,10), dtype = tf.float32, trainable = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "input_X = tf.placeholder(tf.float32, shape = (None, 784))\n",
    "input_y = tf.placeholder(tf.int32, shape = (None,10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "logits = input_X @ W + b\n",
    "probas = tf.nn.softmax(logits) \n",
    "classes = tf.argmax(probas,axis = 1)\n",
    "\n",
    "loss = tf.nn.softmax_cross_entropy_with_logits(labels = input_y, logits = logits) ### cross-entropy loss\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer()\n",
    "step = optimizer.minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "s.run(tf.global_variables_initializer())\n",
    "\n",
    "BATCH_SIZE = 512\n",
    "EPOCHS = 40\n",
    "\n",
    "simpleTrainingCurves = matplotlib_utils.SimpleTrainingCurves(\"cross-entropy\", \"accuracy\")\n",
    "\n",
    "for epoch in range(EPOCHS): \n",
    "    \n",
    "    batch_losses = []\n",
    "    for batch_start in range(0, X_train_flat.shape[0], BATCH_SIZE):  \n",
    "        _, batch_loss = s.run([step, loss], {input_X: X_train_flat[batch_start:batch_start+BATCH_SIZE], \n",
    "                                             input_y: y_train_oh[batch_start:batch_start+BATCH_SIZE]})\n",
    "        batch_losses.append(batch_loss)\n",
    "    \n",
    "    batch_losses[-1] = np.array(list(batch_losses[-1]) + list(np.zeros((176,))))\n",
    "    train_loss = np.mean(batch_losses)\n",
    "    val_loss = s.run(loss, {input_X: X_val_flat, input_y: y_val_oh}\n",
    "    val_loss = np.mean(val_loss)\n",
    "    train_accuracy = accuracy_score(y_train, s.run(classes, {input_X: X_train_flat}))\n",
    "    valid_accuracy = accuracy_score(y_val, s.run(classes, {input_X: X_val_flat}))  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "'''USING MLP''' "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "s = reset_tf_session()\n",
    "input_X = tf.placeholder(tf.float32, shape = (None, 784))\n",
    "input_y = tf.placeholder(tf.int32, shape = (None,10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "hidden_1 = tf.layers.dense(input_X , 256, activation = tf.nn.sigmoid)\n",
    "hidden_2 = tf.layers.dense(hidden_1, 256, activation = tf.nn.sigmoid)\n",
    "logits = tf.layers.dense(hidden_2, 10)\n",
    "probas = tf.layers.dense(hidden_2, 10, activation = tf.nn.sigmoid)\n",
    "classes = tf.argmax(probas, axis = 1)\n",
    "loss = tf.nn.softmax_cross_entropy_with_logits(labels = input_y, logits = logits)\n",
    "optimizer = tf.train.AdamOptimizer()\n",
    "step = optimizer.minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "s.run(tf.global_variables_initializer())\n",
    "\n",
    "BATCH_SIZE = 512\n",
    "EPOCHS = 40\n",
    "\n",
    "simpleTrainingCurves = matplotlib_utils.SimpleTrainingCurves(\"cross-entropy\", \"accuracy\")\n",
    "\n",
    "for epoch in range(EPOCHS):  \n",
    "    \n",
    "    batch_losses = []\n",
    "    for batch_start in range(0, X_train_flat.shape[0], BATCH_SIZE):  \n",
    "        _, batch_loss = s.run([step, loss], {input_X: X_train_flat[batch_start:batch_start+BATCH_SIZE], \n",
    "                                             input_y: y_train_oh[batch_start:batch_start+BATCH_SIZE]})\n",
    "        \n",
    "        batch_losses.append(batch_loss)\n",
    "    \n",
    "    batch_losses[-1] = np.array(list(batch_losses[-1]) + list(np.zeros((176,))))\n",
    "    train_loss = np.mean(batch_losses)\n",
    "    val_loss = s.run(loss, {input_X: X_val_flat, input_y: y_val_oh})  # this part is usually small\n",
    "    val_loss = np.mean(val_loss)\n",
    "    train_accuracy = accuracy_score(y_train, s.run(classes, {input_X: X_train_flat}))  # this is slow and usually skipped\n",
    "    valid_accuracy = accuracy_score(y_val, s.run(classes, {input_X: X_val_flat}))  "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}