Scripts/IDEA_1/COMPRESSION_CUM_CLASSIFICATION_v_2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# This is my first idea for going towards the hybrid architecture that I am looking to achieve. So, according to this idea, I am going to check if it works on the MNIST dataset or not.\n",
    "-------------------------------------------------------------------------------------------------------------------\n",
    "# Technology used: Tensorflow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I start with the usual utility cells for this task"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# packages used for machine learning\n",
    "import tensorflow as tf\n",
    "\n",
    "# packages used for processing: \n",
    "import matplotlib.pyplot as plt # for visualization\n",
    "import numpy as np\n",
    "\n",
    "# for operating system related stuff\n",
    "import os\n",
    "import sys # for memory usage of objects\n",
    "from subprocess import check_output\n",
    "\n",
    "# import pandas for reading the csv files\n",
    "import pandas as pd\n",
    "\n",
    "# to plot the images inline\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# set the random seed to 3 so that the output is repeatable\n",
    "np.random.seed(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Input data files are available in the \"../Data/\" directory.\n",
    "\n",
    "def exec_command(cmd):\n",
    "    '''\n",
    "        function to execute a shell command and see it's \n",
    "        output in the python console\n",
    "        @params\n",
    "        cmd = the command to be executed along with the arguments\n",
    "              ex: ['ls', '../input']\n",
    "    '''\n",
    "    print(check_output(cmd).decode(\"utf8\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data\n",
      "Error_analysis\n",
      "LICENSE\n",
      "Literature_survey\n",
      "Models\n",
      "README.md\n",
      "Res\n",
      "Scripts\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# check the structure of the project directory\n",
    "exec_command(['ls', '../..'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "''' Set the constants for the script '''\n",
    "\n",
    "# various paths of the files\n",
    "data_path = \"../../Data\" # the data path\n",
    "\n",
    "dataset = \"MNIST\"\n",
    "\n",
    "data_files = {\n",
    "    'train': os.path.join(data_path, dataset, \"train.csv\"),\n",
    "    'test' : os.path.join(data_path, dataset, \"test.csv\")\n",
    "}\n",
    "\n",
    "base_model_path = '../../Models'\n",
    "\n",
    "error_analysis_path = '../../Error_analysis/'\n",
    "\n",
    "current_model_path = os.path.join(base_model_path, \"IDEA_1\")\n",
    "\n",
    "model_path_name = os.path.join(current_model_path, \"Model_Error_analysis/Run3\")\n",
    "\n",
    "# constant values:\n",
    "highest_pixel_value = 255\n",
    "train_percentage = 95\n",
    "num_classes = 10\n",
    "no_of_epochs = 500\n",
    "batch_size = 64\n",
    "lr = 0.001\n",
    "hidden_neurons = 512"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Let's load in the data:\n",
    "-------------------------------------------------------------------------------------------------------------------\n",
    "## and perform some basic preprocessing on it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "raw_data = pd.read_csv(data_files['train'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "784 42000\n"
     ]
    }
   ],
   "source": [
    "n_features = len(raw_data.columns) - 1\n",
    "n_examples = len(raw_data.label)\n",
    "print n_features, n_examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>pixel0</th>\n",
       "      <th>pixel1</th>\n",
       "      <th>pixel2</th>\n",
       "      <th>pixel3</th>\n",
       "      <th>pixel4</th>\n",
       "      <th>pixel5</th>\n",
       "      <th>pixel6</th>\n",
       "      <th>pixel7</th>\n",
       "      <th>pixel8</th>\n",
       "      <th>...</th>\n",
       "      <th>pixel774</th>\n",
       "      <th>pixel775</th>\n",
       "      <th>pixel776</th>\n",
       "      <th>pixel777</th>\n",
       "      <th>pixel778</th>\n",
       "      <th>pixel779</th>\n",
       "      <th>pixel780</th>\n",
       "      <th>pixel781</th>\n",
       "      <th>pixel782</th>\n",
       "      <th>pixel783</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10 rows × 785 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   label  pixel0  pixel1  pixel2  pixel3  pixel4  pixel5  pixel6  pixel7  \\\n",
       "0      1       0       0       0       0       0       0       0       0   \n",
       "1      0       0       0       0       0       0       0       0       0   \n",
       "2      1       0       0       0       0       0       0       0       0   \n",
       "3      4       0       0       0       0       0       0       0       0   \n",
       "4      0       0       0       0       0       0       0       0       0   \n",
       "5      0       0       0       0       0       0       0       0       0   \n",
       "6      7       0       0       0       0       0       0       0       0   \n",
       "7      3       0       0       0       0       0       0       0       0   \n",
       "8      5       0       0       0       0       0       0       0       0   \n",
       "9      3       0       0       0       0       0       0       0       0   \n",
       "\n",
       "   pixel8    ...     pixel774  pixel775  pixel776  pixel777  pixel778  \\\n",
       "0       0    ...            0         0         0         0         0   \n",
       "1       0    ...            0         0         0         0         0   \n",
       "2       0    ...            0         0         0         0         0   \n",
       "3       0    ...            0         0         0         0         0   \n",
       "4       0    ...            0         0         0         0         0   \n",
       "5       0    ...            0         0         0         0         0   \n",
       "6       0    ...            0         0         0         0         0   \n",
       "7       0    ...            0         0         0         0         0   \n",
       "8       0    ...            0         0         0         0         0   \n",
       "9       0    ...            0         0         0         0         0   \n",
       "\n",
       "   pixel779  pixel780  pixel781  pixel782  pixel783  \n",
       "0         0         0         0         0         0  \n",
       "1         0         0         0         0         0  \n",
       "2         0         0         0         0         0  \n",
       "3         0         0         0         0         0  \n",
       "4         0         0         0         0         0  \n",
       "5         0         0         0         0         0  \n",
       "6         0         0         0         0         0  \n",
       "7         0         0         0         0         0  \n",
       "8         0         0         0         0         0  \n",
       "9         0         0         0         0         0  \n",
       "\n",
       "[10 rows x 785 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_data.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "labels = np.array(raw_data['label'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(42000,)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# extract the data from the remaining raw_data\n",
    "features = np.ndarray((n_features, n_examples), dtype=np.float32)\n",
    "\n",
    "count = 0 # initialize from zero\n",
    "for pixel in raw_data.columns[1:]:\n",
    "    feature_slice = np.array(raw_data[pixel])\n",
    "    features[count, :] = feature_slice\n",
    "    count += 1 # increment count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(784, 42000)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "features.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Modification: 'don't perform the mean normalization here.'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# normalize the pixel data by dividing the values by the highest_pixel_value\n",
    "# features = features / highest_pixel_value\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADn5JREFUeJzt3X+MHPV5x/HPw3E+xzYQjInjGojxjyZYSDHNxS4NFCgh\nARLFRCIQq0FOROtIxVKs0CrUaVUUpZJbNUG0TVAv2IqTECASIFsK4octWoqaWJyB2OALtrEM2D3u\nQgzYGOOz757+cWN6NjffXe/O7uzd835Jp9udZ2bn0dqfm9mZnfmauwtAPKeU3QCAchB+ICjCDwRF\n+IGgCD8QFOEHgiL8QFCEHwiK8ANBndrMlU2wDp+oyc1cJRDKuzqoAT9s1cxbV/jN7GpJd0pqk3S3\nu69KzT9Rk7XIrqxnlQASNvnGquetebffzNok/UDSNZLmS1piZvNrfT0AzVXPZ/6Fkna6+y53H5B0\nn6TFxbQFoNHqCf9MSa+OeL4nm3YcM1tmZt1m1n1Eh+tYHYAiNfxov7t3uXunu3e2q6PRqwNQpXrC\nv1fSuSOen5NNAzAG1BP+pyXNM7PzzWyCpC9LWl9MWwAareZTfe5+1MyWS3pUw6f61rj7C4V1BqCh\n6jrP7+4PS3q4oF4ANBFf7wWCIvxAUIQfCIrwA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IivAD\nQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANBEX4gqKYO0T1eWUd6JKJDV308WX/lc3Wuf9LR\n3NqOT9+dXLbN0n//V/R2JuuPrl+YrM/u2pVbG3r7YHLZoQMHknXUhy0/EBThB4Ii/EBQhB8IivAD\nQRF+ICjCDwRl7l77wma7JR2QNCjpqLsnTwqfblN9kV1Z8/rKdOrsWbm1F7/7weSyPZetLrib8eGC\nny9P1uf8za+a1Mn4sck3ar/vs2rmLeJLPle4++sFvA6AJmK3Hwiq3vC7pMfMbLOZLSuiIQDNUe9u\n/yXuvtfMPiTpcTP7rbs/OXKG7I/CMkmaqEl1rg5AUera8rv73ux3v6SHJL3vKg9373L3TnfvbFf6\nAhgAzVNz+M1sspmdduyxpM9Ier6oxgA0Vj27/dMlPWRmx17n5+7+SCFdAWi4msPv7rskpS9UH0e2\n/fWHcmt3fvJnyWX7Bg8l69PbPpCs/13/J5L1o0P5O3A9+z+cXHbvW2ck67d89L+S9a+d/mqynvJX\n1zyarP/wDy5L1uf8+bM1rxuc6gPCIvxAUIQfCIrwA0ERfiAowg8EVdclvSdrLF/Sm9J2wbxk/cWV\nU5L1szZMTNan3vN0su5H82/dXa9Tz5mZrPf87TnJ+ovX/bDmdf/ynfRpyLvmza35tcerk7mkly0/\nEBThB4Ii/EBQhB8IivADQRF+ICjCDwTFEN0FGOzZkazPvam+12/eNzFGWfek9HcQll7y303qBEVj\nyw8ERfiBoAg/EBThB4Ii/EBQhB8IivADQXGeH0mvX5x/y3JJWjntF03qBEVjyw8ERfiBoAg/EBTh\nB4Ii/EBQhB8IivADQVU8z29mayR9XlK/u1+YTZsq6X5JsyTtlnSDu7/RuDbRKNbRkawfOjt9C/hn\nB4aS9YsmsH1pVdX8y/xY0tUnTLtN0kZ3nydpY/YcwBhSMfzu/qSkfSdMXixpbfZ4raTrCu4LQIPV\nuk823d17s8evSZpeUD8AmqTuD2Q+PNhf7m3mzGyZmXWbWfcRHa53dQAKUmv4+8xshiRlv/vzZnT3\nLnfvdPfOdqUPLgFonlrDv17S0uzxUknrimkHQLNUDL+Z3SvpV5I+amZ7zOxmSaskXWVmOyR9OnsO\nYAypeJ7f3ZfklK4suBfUqG3aWbm1nlXnJ5f97qUPJeuD/lKyPkHp8/z1HFaaP6EvWd+1Kj0gwtzv\n/Ca3NvTOOzX1NJ7wDQwgKMIPBEX4gaAIPxAU4QeCIvxAUNy6exyw06bk1rZf8x8NXnv6v9CWgcHc\n2hFvSy77iY708ODbbvr3ZP3GPznxYtT/9+Z3Lkgu275hc7I+HrDlB4Ii/EBQhB8IivADQRF+ICjC\nDwRF+IGgOM8/Dgz1v55b+9gTf5Fc9s/mbS+6neO89O2P5dYmvDWQXPZ/Lz0tWd98678l6/fPeSS3\nduk3b0wue8aGZHlcYMsPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0Fxnn8cGDp4MLc29yvPJpd9pehm\nTtCu/Ovic8d4yxy6/uJim8Fx2PIDQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFAVz/Ob2RpJn5fU7+4X\nZtNul/SXkn6XzbbS3R9uVJMYnwY+25msP3DjHRVeob24ZgKqZsv/Y0mjjX5wh7svyH4IPjDGVAy/\nuz8paV8TegHQRPV85l9uZlvMbI2ZnVlYRwCaotbw3yVpjqQFknolfS9vRjNbZmbdZtZ9RIdrXB2A\notUUfnfvc/dBdx+S9CNJCxPzdrl7p7t3tquj1j4BFKym8JvZjBFPvyjp+WLaAdAs1Zzqu1fS5ZKm\nmdkeSf8g6XIzW6DhqzJ3S/p6A3sE0AAVw+/uS0aZvLoBvSCYl69N//e7oJ3z+I3EN/yAoAg/EBTh\nB4Ii/EBQhB8IivADQXHrbiRZ+4Rk/ZQpk5P1nd/KH6L7ikVba+qpWl1vzcqtTV2RvnH4YMG9tCK2\n/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QFOf5gztl0qRkfefdf5isb7us0tXdG06yo+r94M05yfpj\n138ytza4fUfR7Yw5bPmBoAg/EBThB4Ii/EBQhB8IivADQRF+ICjO81fplAvzr0v/7S2nJ5ed8Z/p\nv7FnrHsuWR96991kvW3e7Nza/o+fnVz2w994KVnfNru8u7Q/OzCUrD/2pdyBoiRJgz3bi2xn3GHL\nDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANBVTzPb2bnSvqJpOmSXFKXu99pZlMl3S9plqTdkm5w9zca\n12pjtc09P1m/bf19ubWLOyrc5f0L6fLNK65I1t8c+GCyvnRG/jXzX5hc7j/J5Vu/lFtb+pFfJ5e9\n4/7rkvXztv1PTT1hWDVb/qOSbnX3+ZL+WNItZjZf0m2SNrr7PEkbs+cAxoiK4Xf3Xnd/Jnt8QFKP\npJmSFktam822VlL6zzSAlnJSn/nNbJakiyRtkjTd3Xuz0msa/lgAYIyoOvxmNkXSA5JWuPv+kTV3\ndw0fDxhtuWVm1m1m3Ud0uK5mARSnqvCbWbuGg3+Puz+YTe4zsxlZfYak/tGWdfcud+909852dRTR\nM4ACVAy/mZmk1ZJ63P37I0rrJS3NHi+VtK749gA0SjWX9H5K0k2StprZsWtPV0paJekXZnazpJcl\n3dCYFpvDp3wgWd/27szc2sUdr9S17tXnPVHX8q1s0j+ekVtbtzd9Se55uziV10gVw+/uT0mynPKV\nxbYDoFn4hh8QFOEHgiL8QFCEHwiK8ANBEX4gKG7dnfGe9C2su/41/7rcs795T3LZRl9W2zd4KLd2\n+VPLk8v+U+eDyXolf//TryTr5/26O7d29MhAXetGfdjyA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQ\nNnwHruY43ab6IhubVwEfvH5Rbm3i748klx1c+ftk/eXes5L1aRvTd0Ca9kj+dxQG+0a9wdJ72s48\nM1mvZPCNMXu39nFpk2/Uft+Xdwn+cdjyA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQnOcHxhHO8wOo\niPADQRF+ICjCDwRF+IGgCD8QFOEHgqoYfjM718yeMLNtZvaCmX0jm367me01s+eyn2sb3y6AolQz\naMdRSbe6+zNmdpqkzWb2eFa7w93/pXHtAWiUiuF3915JvdnjA2bWI2lmoxsD0Fgn9ZnfzGZJukjS\npmzScjPbYmZrzGzU+0GZ2TIz6zaz7iM6XFezAIpTdfjNbIqkByStcPf9ku6SNEfSAg3vGXxvtOXc\nvcvdO929s13pe9EBaJ6qwm9m7RoO/j3u/qAkuXufuw+6+5CkH0la2Lg2ARStmqP9Jmm1pB53//6I\n6TNGzPZFSc8X3x6ARqnmaP+nJN0kaauZPZdNWylpiZktkOSSdkv6ekM6BNAQ1Rztf0rSaNcHP1x8\nOwCahW/4AUERfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QFOEHgmrq\nEN1m9jtJL4+YNE3S601r4OS0am+t2pdEb7UqsrePuPvZ1czY1PC/b+Vm3e7eWVoDCa3aW6v2JdFb\nrcrqjd1+ICjCDwRVdvi7Sl5/Sqv21qp9SfRWq1J6K/UzP4DylL3lB1CSUsJvZleb2YtmttPMbiuj\nhzxmttvMtmYjD3eX3MsaM+s3s+dHTJtqZo+b2Y7s96jDpJXUW0uM3JwYWbrU967VRrxu+m6/mbVJ\n2i7pKkl7JD0taYm7b2tqIznMbLekTncv/Zywmf2ppLcl/cTdL8ym/bOkfe6+KvvDeaa7f6tFertd\n0ttlj9ycDSgzY+TI0pKuk/RVlfjeJfq6QSW8b2Vs+RdK2unuu9x9QNJ9khaX0EfLc/cnJe07YfJi\nSWuzx2s1/J+n6XJ6awnu3uvuz2SPD0g6NrJ0qe9doq9SlBH+mZJeHfF8j1pryG+X9JiZbTazZWU3\nM4rp2bDpkvSapOllNjOKiiM3N9MJI0u3zHtXy4jXReOA3/td4u5/JOkaSbdku7ctyYc/s7XS6Zqq\nRm5ullFGln5Pme9drSNeF62M8O+VdO6I5+dk01qCu+/NfvdLekitN/pw37FBUrPf/SX3855WGrl5\ntJGl1QLvXSuNeF1G+J+WNM/MzjezCZK+LGl9CX28j5lNzg7EyMwmS/qMWm/04fWSlmaPl0paV2Iv\nx2mVkZvzRpZWye9dy4147e5N/5F0rYaP+L8k6dtl9JDT12xJv8l+Xii7N0n3ang38IiGj43cLOks\nSRsl7ZC0QdLUFurtp5K2Stqi4aDNKKm3SzS8S79F0nPZz7Vlv3eJvkp53/iGHxAUB/yAoAg/EBTh\nB4Ii/EBQhB8IivADQRF+ICjCDwT1f9QwYr9PSaNKAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f9c39b45dd0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow((features[:, 9]).reshape((28, 28)));"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# use the function to generate the train_dev split\n",
    "-------------------------------------------------------------------------------------------------------------------\n",
    "link -> https://github.com/akanimax/machine-learning-helpers/blob/master/training/data_setup.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# shuffle the data using a random permutation\n",
    "perm = np.random.permutation(n_examples)\n",
    "features = features[:, perm]\n",
    "labels = labels[perm]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7f9c39696b10>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAEVCAYAAAAvoDOaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAESRJREFUeJzt3X2wXHV9x/H3J+GSQBJsQiCGEAgCMmWoBuYaUFDSQXlS\nB2iVmql66SBBB1qZMlVKO5JScDIdeeqA0CABBMqDPAhjGQtFmAxFgcuD4alIwNAQQgKEpyiQkHz7\nx/lduyS7e/fu09nL7/Oa2bnnnN95+O7e/ew5e87u/hQRmFl+xpRdgJmVw+E3y5TDb5Yph98sUw6/\nWaYcfrNMOfxtIOkeSd8oYdlvSVotaZ2k7RuY/zhJ9zazrSrrOl3Sj9qxLiuHw19B0nJJny27jkZI\n6gPOBQ6NiIkR8epm7bMkhaStOrH9iPh+RDT1olUWSbukF8rKW0g6tezaytCRJ4Z1xTRgPPBE2YWM\nFhHxv8DEoXFJuwHLgJtKK6pE3vM3QNJkST+T9LKk19LwzpvNtrukByS9KelWSVMqlj9A0n2SXpf0\na0lzG9zuOEnnS3ox3c5P0z4KPJ1me13SL6osvqSifZ2kT1as9wfpfvxW0hEV0z8k6TJJqyStlHSW\npLE1alsg6eo0PHSU8VeSVqR1f1PSJyQtTff7wopld5f0C0mvSnpF0jWS/qiifT9Jj0h6S9JPJF0v\n6ayK9i9IejSt9z5JH2vk8azi68CSiFje5PKjW0T4lm7AcuCzVaZvD/w5sC0wCfgJ8NOK9nuAlcA+\nwASKPcnVqW0G8CpwJMWL7efS+A4Vy36jRj1nAr8CdgR2AO4D/jm1zQIC2KrGslu0A8cBG4ATgLHA\nt4AXAaX2W4B/S/dhR+AB4MQa619QcR+HtnUJxdHIocA7wE/TemYAa4CD0/x7pMdhXLpfS4DzU9vW\nwPPAt4E+4M+A9cBZqX3ftK79030YSP+3can9h8APG/hfC3gWOK7s511pz/eyC+ilW63wV5lvNvBa\nxfg9wMKK8b3TE3Ys8F3gqs2W/09goGLZWuF/FjiyYvwwYHkabjb8yyrGt03zfJjibcS7wDYV7fOA\nu2usv1r4Z1S0vwr8RcX4TcApNdZ1NPBIGv4MxQupKtrvrQj/xaQXwIr2p4deWEbwv/40sA6YWPbz\nrqyb3/M3QNK2wHnA4cDkNHmSpLERsTGNr6hY5HmKvdZUYFfgy5K+WNHeB9zdwKZ3SuuqXO9OI78H\n7/PS0EBE/F4SFO+Dp6S6VqVpUByprNh8BXWsrhh+u8r4RABJ04ALKAI4KW3ntTTfTsDKSAlNKmvY\nFRiQ9NcV07Zm5I/LAHBTRKwb4XIfGA5/Y04F9gL2j4iXJM0GHqE4dBwys2J4F4rD61conrhXRcQJ\nTWz3RYon+9BJvV3StEaM9OuaKyj2/FMj4r0RLjtS36eo708iYq2ko4GhcwKrgBmSVPECMJPiKGio\nzrMj4uxmNy5pG+DLwDHNruODwCf8ttQnaXzFbSuKvdPbFCfPpgBnVFnuq5L2TkcJZwI3pqOCq4Ev\nSjpM0ti0zrlVThhWcy3wj5J2kDQV+F5aXyNeBjYBH2lk5ohYBdwBnCNpO0lj0om5gxvc3khMojjk\nfkPSDODvKtp+CWwETpa0laSjgDkV7ZcC35S0vwoTJH1e0qQRbP8YiiONRo6+PrAc/i3dThH0odsC\n4HxgG4o9+a+An1dZ7irgCorD6vHA3wBExArgKOB0ikCuoHiyN/LYnwUMAkuBx4CH07RhRcTvgbOB\n/05nxQ9oYLGvUxxCP0kRjhuB6Y1sb4T+CdgPeAP4D+DmoYaIWE9xku944HXgq8DPKI5KiIhBihOW\nF6Yal1GcywBA0iWSLhlm+wMUR2NZ/5iFMr//NgpIuh+4JCIuL7uWDxLv+a3nSDpY0ofTYf8A8DGq\nH21ZC3zCz3rRXsANFJ83eA74UjonYW3kw36zTPmw3yxTDr9Zphx+s0w5/GaZcvjNMuXwm2XK4TfL\nlMNvlimH3yxTDr9Zphx+s0w5/GaZcvjNMuXwm2Wqq9/n31rjYjwTurlJs6y8w+9YH+9q+DlbDL+k\nwyl+gnks8KOIWFhv/vFMYH8d0somzayO++Ouhudt+rA/deN0EXAERScV8yTt3ez6zKy7WnnPP4ei\n95fn0i+uXkfxK7VmNgq0Ev4ZvL8nlRfStPeRNF/SoKTBDcWvL5tZD+j42f6IWBQR/RHR38e4Tm/O\nzBrUSvhX8v4uqnZO08xsFGgl/A8Ce0raTdLWwFeA29pTlpl1WtOX+iLiPUknU3Q3PRZYHBFPDLOY\nmfWIlq7zR8TtFH3bmdko44/3mmXK4TfLlMNvlimH3yxTDr9Zphx+s0w5/GaZcvjNMuXwm2XK4TfL\nlMNvlimH3yxTDr9Zphx+s0w5/GaZcvjNMuXwm2XK4TfLlMNvlimH3yxTDr9Zphx+s0w5/GaZcvjN\nMuXwm2XK4TfLlMNvlimH3yxTDr9Zphx+s0y11EW3pOXAW8BG4L2I6G9HUWbWeS2FP/nTiHilDesx\nsy7yYb9ZploNfwB3SHpI0vxqM0iaL2lQ0uAG3m1xc2bWLq0e9h8UESsl7QjcKel/ImJJ5QwRsQhY\nBLCdpkSL2zOzNmlpzx8RK9PfNcAtwJx2FGVmndd0+CVNkDRpaBg4FHi8XYWZWWe1ctg/DbhF0tB6\n/j0ift6Wqsys45oOf0Q8B3y8jbWYWRf5Up9Zphx+s0w5/GaZcvjNMuXwm2XK4TfLlMNvlimH3yxT\nDr9Zphx+s0w5/GaZcvjNMuXwm2WqHT/gmYX4VO0vML6217ZdrGRLm7au3fbA9y7qXiFVjFXt/cvG\n2FR32X3PPblu+/Rz7muqJit4z2+WKYffLFMOv1mmHH6zTDn8Zply+M0y5fCbZcrX+Rv0zHG1L6b/\n5vMXdrGSkal/Jb0L24+NTS/7tyfcWLf92nN2anrd5j2/WbYcfrNMOfxmmXL4zTLl8JtlyuE3y5TD\nb5YpX+dv0DbP99Vse2Xj23WXnTp2m3aXY9ayYff8khZLWiPp8YppUyTdKemZ9HdyZ8s0s3Zr5LD/\nCuDwzaadBtwVEXsCd6VxMxtFhg1/RCwB1m42+SjgyjR8JXB0m+sysw5r9j3/tIhYlYZfAqbVmlHS\nfGA+wHjK/a07M/t/LZ/tj4gAok77oojoj4j+Psa1ujkza5Nmw79a0nSA9HdN+0oys25oNvy3AQNp\neAC4tT3lmFm3DPueX9K1wFxgqqQXgDOAhcANko4HngeO7WSRvWDm2bV/I/6wd75Td9n127W7mtHj\nloEf1Gzbo89vA8s0bPgjYl6NpkPaXIuZdZE/3muWKYffLFMOv1mmHH6zTDn8ZpnyV3rbwF1F17bi\nLz9Us22Pvne6WIltznt+s0w5/GaZcvjNMuXwm2XK4TfLlMNvlimH3yxTDr9Zphx+s0w5/GaZcvjN\nMuXwm2XK4TfLlMNvlimH3yxT/j6/teS3Cz9Zt/0T4+r91oF/urtM3vObZcrhN8uUw2+WKYffLFMO\nv1mmHH6zTDn8ZplqpIvuxcAXgDURsU+atgA4AXg5zXZ6RNzeqSKtPGP/eM+67QfOfbxu+8Qxvpbf\nqxrZ818BHF5l+nkRMTvdHHyzUWbY8EfEEmBtF2oxsy5q5T3/yZKWSlosaXLbKjKzrmg2/BcDuwOz\ngVXAObVmlDRf0qCkwQ282+TmzKzdmgp/RKyOiI0RsQm4FJhTZ95FEdEfEf19/iKHWc9oKvySpleM\nHgPUP+VrZj2nkUt91wJzgamSXgDOAOZKmg0EsBw4sYM1mlkHDBv+iJhXZfJlHajFetDrH9++bvut\nM69ret3rNtU/B3TBv36pbvuO1PutABuOP+FnlimH3yxTDr9Zphx+s0w5/GaZcvjNMuWf7raOemT9\nppptJ515at1ld7zcl/I6yXt+s0w5/GaZcvjNMuXwm2XK4TfLlMNvlimH3yxTvs5vde100rKWlr9+\n7f4126Zc/suW1m2t8Z7fLFMOv1mmHH6zTDn8Zply+M0y5fCbZcrhN8uUr/NnbtOn963bfumsi4ZZ\ng3thGq285zfLlMNvlimH3yxTDr9Zphx+s0w5/GaZcvjNMjXsdX5JM4EfA9OAABZFxAWSpgDXA7OA\n5cCxEfFa50q1Tlj7nd/VbZ84xtfxP6ga2fO/B5waEXsDBwAnSdobOA24KyL2BO5K42Y2Sgwb/ohY\nFREPp+G3gKeAGcBRwJVptiuBoztVpJm134je80uaBewL3A9Mi4hVqeklircFZjZKNBx+SROBm4BT\nIuLNyraICIrzAdWWmy9pUNLgBt5tqVgza5+Gwi+pjyL410TEzWnyaknTU/t0YE21ZSNiUUT0R0R/\nn78EYtYzhg2/JAGXAU9FxLkVTbcBA2l4ALi1/eWZWac08pXeA4GvAY9JejRNOx1YCNwg6XjgeeDY\nzpRoZp0wbPgj4l5ANZoPaW85ZtYt/oSfWaYcfrNMOfxmmXL4zTLl8JtlyuE3y5TDb5Yph98sUw6/\nWaYcfrNMOfxmmXL4zTLl8JtlyuE3y5S76M7c2jXb1W3fxKa67WOG2X888vf71WzrY7DustZZ3vOb\nZcrhN8uUw2+WKYffLFMOv1mmHH6zTDn8Zpnydf7MffT4+tfa31ixvm775DHj67Zvs+yVmm3v1V3S\nOs17frNMOfxmmXL4zTLl8JtlyuE3y5TDb5Yph98sU8OGX9JMSXdLelLSE5K+naYvkLRS0qPpdmTn\nyzWzdmnkQz7vAadGxMOSJgEPSboztZ0XET/oXHlm1inDhj8iVgGr0vBbkp4CZnS6MDPrrBG955c0\nC9gXuD9NOlnSUkmLJU2uscx8SYOSBjfwbkvFmln7NBx+SROBm4BTIuJN4GJgd2A2xZHBOdWWi4hF\nEdEfEf19jGtDyWbWDg2FX1IfRfCviYibASJidURsjIhNwKXAnM6VaWbt1sjZfgGXAU9FxLkV06dX\nzHYM8Hj7yzOzTmnkbP+BwNeAxyQ9mqadDsyTNBsIYDlwYkcqNLOOaORs/72AqjTd3v5yzKxb/Ak/\ns0w5/GaZcvjNMuXwm2XK4TfLlMNvlin/dLfV9anrT63b/tS8i7pUibWb9/xmmXL4zTLl8JtlyuE3\ny5TDb5Yph98sUw6/WaYUEd3bmPQy8HzFpKlA7T6cy9WrtfVqXeDamtXO2naNiB0ambGr4d9i49Jg\nRPSXVkAdvVpbr9YFrq1ZZdXmw36zTDn8ZpkqO/yLSt5+Pb1aW6/WBa6tWaXUVup7fjMrT9l7fjMr\nSSnhl3S4pKclLZN0Whk11CJpuaTHUs/DgyXXsljSGkmPV0ybIulOSc+kv1W7SSuptp7oublOz9Kl\nPna91uN11w/7JY0FfgN8DngBeBCYFxFPdrWQGiQtB/ojovRrwpI+A6wDfhwR+6Rp/wKsjYiF6YVz\nckR8t0dqWwCsK7vn5tShzPTKnqWBo4HjKPGxq1PXsZTwuJWx558DLIuI5yJiPXAdcFQJdfS8iFgC\nrN1s8lHAlWn4SoonT9fVqK0nRMSqiHg4Db8FDPUsXepjV6euUpQR/hnAiorxF+itLr8DuEPSQ5Lm\nl11MFdNSt+kALwHTyiymimF7bu6mzXqW7pnHrpker9vNJ/y2dFBE7AccAZyUDm97UhTv2Xrpck1D\nPTd3S5Wepf+gzMeu2R6v262M8K8EZlaM75ym9YSIWJn+rgFuofd6H1491Elq+rum5Hr+oJd6bq7W\nszQ98Nj1Uo/XZYT/QWBPSbtJ2hr4CnBbCXVsQdKEdCIGSROAQ+m93odvAwbS8ABwa4m1vE+v9Nxc\nq2dpSn7seq7H64jo+g04kuKM/7PAP5RRQ426PgL8Ot2eKLs24FqKw8ANFOdGjge2B+4CngH+C5jS\nQ7VdBTwGLKUI2vSSajuI4pB+KfBouh1Z9mNXp65SHjd/ws8sUz7hZ5Yph98sUw6/WaYcfrNMOfxm\nmXL4zTLl8JtlyuE3y9T/AVd5MWdxmtTPAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f9c39707250>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "random_index = np.random.randint(n_examples)\n",
    "random_image = features[:, random_index].reshape((28, 28))\n",
    "# use plt to plot the image\n",
    "plt.figure().suptitle(\"Label of the image: \" + str(labels[random_index]))\n",
    "plt.imshow(random_image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# function to split the data into train - dev sets:\n",
    "def split_train_dev(X, Y, train_percentage):\n",
    "    '''\n",
    "        function to split the given data into two small datasets (train - dev)\n",
    "        @param\n",
    "        X, Y => the data to be split\n",
    "        (** Make sure the train dimension is the first one)\n",
    "        train_percentage => the percentage which should be in the training set.\n",
    "        (**this should be in 100% not decimal)\n",
    "        @return => train_X, train_Y, test_X, test_Y\n",
    "    '''\n",
    "    m_examples = len(X)\n",
    "    assert train_percentage < 100, \"Train percentage cannot be greater than 100! NOOB!\"\n",
    "    partition_point = int((m_examples * (float(train_percentage) / 100)) + 0.5) # 0.5 is added for rounding\n",
    "\n",
    "    # construct the train_X, train_Y, test_X, test_Y sets:\n",
    "    train_X = X[: partition_point]; train_Y = Y[: partition_point]\n",
    "    test_X  = X[partition_point: ]; test_Y  = Y[partition_point: ]\n",
    "\n",
    "    assert len(train_X) + len(test_X) == m_examples, \"Something wrong in X splitting\"\n",
    "    assert len(train_Y) + len(test_Y) == m_examples, \"Something wrong in Y splitting\"\n",
    "\n",
    "    # return the constructed sets\n",
    "\n",
    "    return train_X, train_Y, test_X, test_Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_X, train_Y, test_X, test_Y = split_train_dev(features.T, labels, train_percentage)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((39900, 784), (39900,), (2100, 784), (2100,))"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_X.shape, train_Y.shape, test_X.shape, test_Y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((784, 39900), (784, 2100))"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_X = train_X.T; test_X = test_X.T\n",
    "train_X.shape, test_X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7f9c395691d0>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAEVCAYAAAAvoDOaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAE6hJREFUeJzt3Xm0HHWZxvHvQ8hCQtAkQAiBIQrBERkJTABRBvCobKOA\nyiKjTBjBoMIgHnT04MIy6MlRWRQVCJIREVFkHwYdMKgMIpHLYthGSDCQhCwQQMIWsrzzR1WcTrj9\nu517u7v63t/zOadPuuvtqnpvp59b1VW366eIwMzys1HVDZhZNRx+s0w5/GaZcvjNMuXwm2XK4TfL\nlMPfBJJ+I+n4Cub9lKQlkl6UNKaB5x8r6Y7erKubZZ0m6QfNWJZVw+GvIWmepPdW3UcjJA0GzgX2\nj4hNI2LZevUJkkLSxq1Yf0R8PSJ69UurSpImSfofSX+RtEDSV6ruqSoOf/81FhgGPFR1I/3MT4Db\ngdHAvsCnJR1SbUvVcPgbIGmUpJskPS3pufL+Nus9bXtJf5D0gqQbJI2umf8dku6U9LykP0rar8H1\nDpV0vqSnytv55bQdgT+VT3te0m3dzH57Tf1FSXvVLPdb5c/xZ0kH1Ux/g6RLJS2StFDS2ZIG1ent\nDEk/Lu+v3cv4F0nzy2V/UtLukmaXP/d3a+bdXtJtkpZJekbSFZLeWFPfTdJ9kpZL+rmkn0k6u6b+\nfkn3l8u9U9LbG3k9SxOAKyJidUTMBe4A3rYB8w8cEeFbeQPmAe/tZvoY4MPAcGAk8HPg+pr6b4CF\nwM7ACOAa4MdlbTywDDiY4pft+8rHW9TMe3ydfs4C7gK2BLYA7gT+vaxNAALYuM68r6sDxwIrgU8A\ng4BPAU8BKuvXAReXP8OWwB+AE+os/4yan3Htui6i2BvZH3gVuL5cznhgKbBv+fwdytdhaPlz3Q6c\nX9aGAE8AnwEGAx8CXgPOLuu7lsvas/wZppT/b0PL+veB7yf+j78OTCuX/RZgAbB71e+9St7vVTfQ\nSbd64e/meZOA52oe/waYVvN4p/INOwj4AnD5evP/NzClZt564Z8LHFzz+ABgXnm/t+GfU/N4ePmc\nrSg+RqwANqmpHw38us7yuwv/+Jr6MuComsfXAKfUWdZhwH3l/X0ofpGqpn5HTfgvpPwFWFP/09pf\nLA38370TmAOsKns+s+r3XVW3lhwMGmgkDQfOAw4ERpWTR0oaFBGry8fza2Z5gmLLsjmwHXCEpA/U\n1AcDv25g1VuXy6pd7tYb/hOsY/HaOxHxsiSATSk+Aw8GFpXToNhTmb/+AhKW1Nx/pZvHmwJIGgt8\nG/gHij2pjYDnyudtDSyMMqml2h62A6ZI+teaaUNo4HUpP4r9EjiJ4rP/VsDVkpZExPd7/OkGGH/m\nb8ypFLuIe0bEZhRbJwDVPGfbmvt/Q7F7/QzFG/fyiHhjzW1ERExrYL1PUbzZa5f7VIM9b+jXNedT\nbPk3r+lzs4hoxefhr1P093fl6/kx/v+1XASMV81vINZ9becDX1vv9RweEVc2sN43A6sj4kcRsSoi\nFgA/pfhIlh2H//UGSxpWc9uYYuv0CsXBs9HA6d3M9zFJO5V7CWcBV5d7BT8GPiDpAEmDymXu180B\nw+5cCXxZ0haSNge+Wi6vEU8Dayje8D2KiEXALcA5kjaTtFF5YG7fBte3IUYCLwJ/kTQe+HxN7ffA\nauAkSRtLOhTYo6Z+CfBJSXuqMELSP0oa2cB6HwUk6Z/Kn28r4ChgdlN+qn7G4X+9mymCvvZ2BnA+\nsAnFlvwuil3H9V0O/JBit3oYcDJARMwHDgVOowjkfIo3eyOv/dlAF8Wb8wHg3nJajyLiZeBrwO/K\no+LvaGC2f6bYhX6YYjf8amBcI+vbQGcCuwF/Af4LuHZtISJeozjIdxzwPMVewU0UeyVERBfFAcvv\nlj3OoTiWAYCkiyRd1N1KI+KFctmfLee9H3iQBl/TgUbrfrQy6zySZgEXRcR/VN3LQOItv3UcSftK\n2qrc7Z8CvJ3u97asD3y03zrRW4CrKP7e4HHg8PKYhDWRd/vNMuXdfrNMOfxmmXL4zTLl8JtlyuE3\ny5TDb5Yph98sUw6/WaYcfrNMOfxmmXL4zTLl8JtlyuE3y5TDb5aptn6ff4iGxjBGtHOVZll5lZd4\nLVao52f2MfySDqS4BPMg4Ac9XZF2GCPYU+/pyyrNLGFWzGz4ub3e7S+HcfoecBDFIBVHS9qpt8sz\ns/bqy2f+PShGf3m8vOLqTymuUmtm/UBfwj+edUdSWVBOW4ekqZK6JHWtLK6+bGYdoOVH+yNiekRM\njojJgxna6tWZWYP6Ev6FrDuM0jblNDPrB/oS/ruBiZLeJGkI8BHgxua0ZWat1utTfRGxStJJFMNN\nDwJmRMRDTevMzFqqT+f5I+JmirHtzKyf8Z/3mmXK4TfLlMNvlimH3yxTDr9Zphx+s0w5/GaZcvjN\nMuXwm2XK4TfLlMNvlimH3yxTDr9Zphx+s0w5/GaZcvjNMuXwm2XK4TfLlMNvlimH3yxTDr9Zpto6\nRLdZp1hy8juT9Tcf/liy/tI+TzeznUp4y2+WKYffLFMOv1mmHH6zTDn8Zply+M0y5fCbZcrn+W3A\n0tChdWs7HfVIct5d3/Bksn7byK2T9TXLlyfrnaBP4Zc0D1gOrAZWRcTkZjRlZq3XjC3/uyPimSYs\nx8zayJ/5zTLV1/AHcIukeyRN7e4JkqZK6pLUtZIVfVydmTVLX3f7946IhZK2BG6V9L8RcXvtEyJi\nOjAdYDONjj6uz8yapE9b/ohYWP67FLgO2KMZTZlZ6/U6/JJGSBq59j6wP/Bgsxozs9bqy27/WOA6\nSWuX85OI+GVTurKOsfGbtuvT/KufWly3Fitaewzoyc//fd3af064IDnv/g9/KFkfsvyJXvXUSXod\n/oh4HNilib2YWRv5VJ9Zphx+s0w5/GaZcvjNMuXwm2XKX+kd4J6Zuleyvu1HH0/Wr9nhumR9Dek/\n2nzLrd3+1TcAE4+9JzlvT148Ys9k/fpPfLNu7crl6VOYw09Qsr4qWe0fvOU3y5TDb5Yph98sUw6/\nWaYcfrNMOfxmmXL4zTLl8/z9wMbj05eJfvjL29StzTnke31a9xFzD0jW10QP24+Xev8W6+nnfveX\n7kzWRyda+845RyTnHfP475P1gcBbfrNMOfxmmXL4zTLl8JtlyuE3y5TDb5Yph98sUz7P3w+kzuMD\nPHrIhXVrc1e9mpz3o2d+LlkfPaNv57snsqTX8y67ZHiyfvoW9yfrO95ycv3aJQP/PH5PvOU3y5TD\nb5Yph98sUw6/WaYcfrNMOfxmmXL4zTLl8/wd4Plj0tfW7+k7+XetqH+N+RPP+3xy3rEz0t+J76vU\nd/J1Rfqa/7+beFWyfsy89ybrO168MlnPXY9bfkkzJC2V9GDNtNGSbpX0WPnvqNa2aWbN1shu/w+B\nA9eb9kVgZkRMBGaWj82sH+kx/BFxO/DsepMPBS4r718GHNbkvsysxXr7mX9sRCwq7y8GxtZ7oqSp\nwFSAYaT/VtvM2qfPR/sjIqD+aI0RMT0iJkfE5MEM7evqzKxJehv+JZLGAZT/Lm1eS2bWDr0N/43A\nlPL+FOCG5rRjZu3S42d+SVcC+wGbS1oAnA5MA66SdBzwBHBkK5sc6FaMSo8F35NfLd+5bm3sBa09\nj9+T1Ln86ybelJz30wv3TtafPSV9nQPunp2uZ67H8EfE0XVK72lyL2bWRv7zXrNMOfxmmXL4zTLl\n8JtlyuE3y5S/0jsA7DCs/uWxZ43ZJTnv6mXrf21jwyy+/q3J+r0Tr6hbm/nKsOS8Tx6/XbIesx9I\n1i3NW36zTDn8Zply+M0y5fCbZcrhN8uUw2+WKYffLFM+zz8AHLlp/Wup/Pzawcl5X/ryrsn63MPT\nV196cPJ3kvU1DKpb+8pZxyfnfeNsD6PdSt7ym2XK4TfLlMNvlimH3yxTDr9Zphx+s0w5/GaZUjHg\nTntsptGxp3zR3/VtvFXd0c4A2OUXi5L1M7e8r5ntrGMj0pcVn7vqlWT9gNtOrlt76+f+nJy3r9ca\nyNGsmMkL8WxD14L3lt8sUw6/WaYcfrNMOfxmmXL4zTLl8JtlyuE3y1QjQ3TPAN4PLI2InctpZwCf\nAJ4un3ZaRNzcqiYHulWL6193H6DrpN2S9WeurD8M9+aDNulVT2vNfCX9ff5zj56SrO949z11a6t7\n1ZE1SyNb/h8CB3Yz/byImFTeHHyzfqbH8EfE7YD/1MpsgOnLZ/6TJM2WNEPSqKZ1ZGZt0dvwXwhs\nD0wCFgHn1HuipKmSuiR1rWRFL1dnZs3Wq/BHxJKIWB0Ra4BLgD0Sz50eEZMjYvJg0gePzKx9ehV+\nSeNqHn4QeLA57ZhZuzRyqu9KYD9gc0kLgNOB/SRNAgKYB5zQwh7NrAX8ff4B4NGLd69bm/P+i/u0\n7Hf98chkfdThTyXra15+uU/rtw3j7/ObWY8cfrNMOfxmmXL4zTLl8JtlyuE3y5SH6O4Hnv34Xsn6\nbw/6Zt3aA6+lh+h+25D0W+B3u1yVrB+y5WHJ+pp5TybrVh1v+c0y5fCbZcrhN8uUw2+WKYffLFMO\nv1mmHH6zTPk8fweIvXZJ1i/96nnJ+t2vbl239u1/+0hy3qsvSC97zEbpS3/H8GHJunUub/nNMuXw\nm2XK4TfLlMNvlimH3yxTDr9Zphx+s0z5PH8bbDRyZLI+dNriZP1tg4ck66dO+XDd2ia//UNy3qmf\n/VCyfs0Ov0jW5x49Jlmf8JVk2SrkLb9Zphx+s0w5/GaZcvjNMuXwm2XK4TfLlMNvlqkez/NL2hb4\nETAWCGB6RHxb0mjgZ8AEYB5wZEQ817pW+6+nj9o5WZ+1w/eS9SPmHpCsb/Tb++rXhg9PzrvzG9JD\nbA9Sevuww6ULk/VVyapVqZEt/yrg1IjYCXgHcKKknYAvAjMjYiIws3xsZv1Ej+GPiEURcW95fznw\nCDAeOBS4rHzaZUB66BYz6ygb9Jlf0gRgV2AWMDYiFpWlxRQfC8ysn2g4/JI2Ba4BTomIF2prEREU\nxwO6m2+qpC5JXStZ0admzax5Ggq/pMEUwb8iIq4tJy+RNK6sjwOWdjdvREyPiMkRMXkwQ5vRs5k1\nQY/hlyTgUuCRiDi3pnQjMKW8PwW4ofntmVmrNPKV3ncBxwAPSLq/nHYaMA24StJxwBPAka1psf9b\nPiFdX9P9J6a/evLyHZL1MTxdt7b445OS856+xQXJ+tyVryTrrPTJvP6qx/BHxB2A6pTf09x2zKxd\n/Bd+Zply+M0y5fCbZcrhN8uUw2+WKYffLFO+dHcbDHq13pnSxizbI30ufdnuu9et3XngN3pYenoI\n7vfddGqyPnHhrB6Wb53KW36zTDn8Zply+M0y5fCbZcrhN8uUw2+WKYffLFM+z98GEy5+LFn/9Af2\nSdbnHHxxsp66HsCS1clZ+durTkzWJ372rvQCrN/ylt8sUw6/WaYcfrNMOfxmmXL4zTLl8JtlyuE3\ny5SKkbbaYzONjj3lq32btcqsmMkL8WxDF5Dwlt8sUw6/WaYcfrNMOfxmmXL4zTLl8JtlyuE3y1SP\n4Ze0raRfS3pY0kOSPlNOP0PSQkn3l7eDW9+umTVLIxfzWAWcGhH3ShoJ3CPp1rJ2XkR8q3XtmVmr\n9Bj+iFgELCrvL5f0CDC+1Y2ZWWtt0Gd+SROAXYG1YzSdJGm2pBmSRtWZZ6qkLkldK1nRp2bNrHka\nDr+kTYFrgFMi4gXgQmB7YBLFnsE53c0XEdMjYnJETB7M0Ca0bGbN0FD4JQ2mCP4VEXEtQEQsiYjV\nEbEGuATYo3VtmlmzNXK0X8ClwCMRcW7N9HE1T/sg8GDz2zOzVmnkaP+7gGOAByTdX047DTha0iQg\ngHnACS3p0MxaopGj/XcA3X0/+Obmt2Nm7eK/8DPLlMNvlimH3yxTDr9Zphx+s0w5/GaZcvjNMuXw\nm2XK4TfLlMNvlimH3yxTDr9Zphx+s0w5/GaZausQ3ZKeBp6ombQ58EzbGtgwndpbp/YF7q23mtnb\ndhGxRSNPbGv4X7dyqSsiJlfWQEKn9tapfYF7662qevNuv1mmHH6zTFUd/ukVrz+lU3vr1L7AvfVW\nJb1V+pnfzKpT9ZbfzCpSSfglHSjpT5LmSPpiFT3UI2mepAfKkYe7Ku5lhqSlkh6smTZa0q2SHiv/\n7XaYtIp664iRmxMjS1f62nXaiNdt3+2XNAh4FHgfsAC4Gzg6Ih5uayN1SJoHTI6Iys8JS9oHeBH4\nUUTsXE77BvBsREwrf3GOiogvdEhvZwAvVj1yczmgzLjakaWBw4BjqfC1S/R1JBW8blVs+fcA5kTE\n4xHxGvBT4NAK+uh4EXE78Ox6kw8FLivvX0bx5mm7Or11hIhYFBH3lveXA2tHlq70tUv0VYkqwj8e\nmF/zeAGdNeR3ALdIukfS1Kqb6cbYcth0gMXA2Cqb6UaPIze303ojS3fMa9ebEa+bzQf8Xm/viNgN\nOAg4sdy97UhRfGbrpNM1DY3c3C7djCz9V1W+dr0d8brZqgj/QmDbmsfblNM6QkQsLP9dClxH540+\nvGTtIKnlv0sr7uevOmnk5u5GlqYDXrtOGvG6ivDfDUyU9CZJQ4CPADdW0MfrSBpRHohB0ghgfzpv\n9OEbgSnl/SnADRX2so5OGbm53sjSVPzaddyI1xHR9htwMMUR/7nAl6rooU5fbwb+WN4eqro34EqK\n3cCVFMdGjgPGADOBx4BfAaM7qLfLgQeA2RRBG1dRb3tT7NLPBu4vbwdX/dol+qrkdfNf+Jllygf8\nzDLl8JtlyuE3y5TDb5Yph98sUw6/WaYcfrNMOfxmmfo/jpf7cwJ4YDYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f9c396d5490>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# check by plotting some image\n",
    "random_index = np.random.randint(train_X.shape[-1])\n",
    "random_image = train_X[:, random_index].reshape((28, 28))\n",
    "# use plt to plot the image\n",
    "plt.figure().suptitle(\"Label of the image: \" + str(train_Y[random_index]))\n",
    "plt.imshow(random_image)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Point to reset from here onwards: "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# defining the Tensorflow graph for this task:\n",
    "tf.reset_default_graph() # reset the graph here:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# define the placeholders:\n",
    "tf_input_pixels = tf.placeholder(tf.float32, shape=(n_features, None))\n",
    "tf_integer_labels = tf.placeholder(tf.int32, shape=(None,))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# image shaped pixels for the input_pixels:\n",
    "tf_input_images = tf.reshape(tf.transpose(tf_input_pixels), shape=(-1, 28, 28, 1))\n",
    "input_image_summary = tf.summary.image(\"input_image\", tf_input_images)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'one_hot:0' shape=(10, ?) dtype=float32>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# define the one hot encoded version fo the integer_labels\n",
    "tf_one_hot_encoded_labels = tf.one_hot(tf_integer_labels, depth=num_classes, axis=0)\n",
    "tf_one_hot_encoded_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# define the layer 0 biases:\n",
    "lay_0_b = tf.get_variable(\"layer_0_biases\", shape=(n_features, 1), initializer=tf.zeros_initializer())\n",
    "\n",
    "\n",
    "# layer 1 weights \n",
    "lay_1_W = tf.get_variable(\"layer_1_weights\", shape=(hidden_neurons, n_features), \n",
    "                              dtype=tf.float32, initializer=tf.contrib.layers.xavier_initializer())\n",
    "lay_1_b = tf.get_variable(\"layer_1_biases\", shape=(hidden_neurons, 1), \n",
    "                            dtype=tf.float32, initializer=tf.zeros_initializer())\n",
    "\n",
    "# layer 2 weights\n",
    "lay_2_W = tf.get_variable(\"layer_2_weights\", shape=(hidden_neurons, hidden_neurons), \n",
    "                              dtype=tf.float32, initializer=tf.contrib.layers.xavier_initializer())\n",
    "lay_2_b = tf.get_variable(\"layer_2_biases\", shape=(hidden_neurons, 1), \n",
    "                            dtype=tf.float32, initializer=tf.zeros_initializer())\n",
    "\n",
    "# layer 3 weights\n",
    "lay_3_W = tf.get_variable(\"layer_3_weights\", shape=(hidden_neurons, hidden_neurons), \n",
    "                              dtype=tf.float32, initializer=tf.contrib.layers.xavier_initializer())\n",
    "lay_3_b = tf.get_variable(\"layer_3_biases\", shape=(hidden_neurons, 1), \n",
    "                            dtype=tf.float32, initializer=tf.zeros_initializer())\n",
    "\n",
    "# layer 4 weights\n",
    "lay_4_W = tf.get_variable(\"layer_4_weights\", shape=(hidden_neurons, hidden_neurons), \n",
    "                              dtype=tf.float32, initializer=tf.contrib.layers.xavier_initializer())\n",
    "lay_4_b = tf.get_variable(\"layer_4_biases\", shape=(hidden_neurons, 1), \n",
    "                            dtype=tf.float32, initializer=tf.zeros_initializer())\n",
    "\n",
    "# layer 5 weights\n",
    "lay_5_W = tf.get_variable(\"layer_5_weights\", shape=(hidden_neurons, hidden_neurons), \n",
    "                              dtype=tf.float32, initializer=tf.contrib.layers.xavier_initializer())\n",
    "lay_5_b = tf.get_variable(\"layer_5_biases\", shape=(hidden_neurons, 1), \n",
    "                            dtype=tf.float32, initializer=tf.zeros_initializer())\n",
    "\n",
    "# layer 6 weights\n",
    "lay_6_W = tf.get_variable(\"layer_6_weights\", shape=(hidden_neurons, hidden_neurons), \n",
    "                              dtype=tf.float32, initializer=tf.contrib.layers.xavier_initializer())\n",
    "lay_6_b = tf.get_variable(\"layer_6_biases\", shape=(hidden_neurons, 1), \n",
    "                            dtype=tf.float32, initializer=tf.zeros_initializer())\n",
    "\n",
    "# layer 7 weights\n",
    "lay_7_W = tf.get_variable(\"layer_7_weights\", shape=(hidden_neurons, hidden_neurons), \n",
    "                              dtype=tf.float32, initializer=tf.contrib.layers.xavier_initializer())\n",
    "lay_7_b = tf.get_variable(\"layer_7_biases\", shape=(hidden_neurons, 1), \n",
    "                            dtype=tf.float32, initializer=tf.zeros_initializer())\n",
    "\n",
    "# layer 8 weights\n",
    "lay_8_W = tf.get_variable(\"layer_8_weights\", shape=(hidden_neurons, hidden_neurons), \n",
    "                              dtype=tf.float32, initializer=tf.contrib.layers.xavier_initializer())\n",
    "lay_8_b = tf.get_variable(\"layer_8_biases\", shape=(hidden_neurons, 1), \n",
    "                            dtype=tf.float32, initializer=tf.zeros_initializer())\n",
    "\n",
    "# layer 9 weights\n",
    "lay_9_W = tf.get_variable(\"layer_9_weights\", shape=(hidden_neurons, hidden_neurons), \n",
    "                              dtype=tf.float32, initializer=tf.contrib.layers.xavier_initializer())\n",
    "lay_9_b = tf.get_variable(\"layer_9_biases\", shape=(hidden_neurons, 1), \n",
    "                            dtype=tf.float32, initializer=tf.zeros_initializer())\n",
    "\n",
    "# layer 10 weights\n",
    "lay_10_W = tf.get_variable(\"layer_10_weights\", shape=(hidden_neurons, hidden_neurons), \n",
    "                              dtype=tf.float32, initializer=tf.contrib.layers.xavier_initializer())\n",
    "lay_10_b = tf.get_variable(\"layer_10_biases\", shape=(hidden_neurons, 1), \n",
    "                            dtype=tf.float32, initializer=tf.zeros_initializer())\n",
    "\n",
    "# layer 11 weights\n",
    "lay_11_W = tf.get_variable(\"layer_11_weights\", shape=(hidden_neurons, hidden_neurons), \n",
    "                              dtype=tf.float32, initializer=tf.contrib.layers.xavier_initializer())\n",
    "lay_11_b = tf.get_variable(\"layer_11_biases\", shape=(hidden_neurons, 1), \n",
    "                            dtype=tf.float32, initializer=tf.zeros_initializer())\n",
    "\n",
    "# layer 12 weights\n",
    "lay_12_W = tf.get_variable(\"layer_12_weights\", shape=(num_classes, hidden_neurons), \n",
    "                              dtype=tf.float32, initializer=tf.contrib.layers.xavier_initializer())\n",
    "lay_12_b = tf.get_variable(\"layer_12_biases\", shape=(num_classes, 1), \n",
    "                            dtype=tf.float32, initializer=tf.zeros_initializer())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# define the forward computation:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# forward computation:\n",
    "z1 = tf.matmul(lay_1_W, tf_input_pixels) + lay_1_b\n",
    "a1 = tf.abs(z1)\n",
    "\n",
    "z2 = tf.matmul(lay_2_W, a1) + lay_2_b\n",
    "a2 = tf.abs(z2)\n",
    "\n",
    "z3 = tf.matmul(lay_3_W, a2) + lay_3_b\n",
    "a3 = tf.abs(z3) \n",
    "\n",
    "z4 = tf.matmul(lay_4_W, a3) + lay_4_b\n",
    "a4 = tf.abs(z4) \n",
    "\n",
    "z5 = tf.matmul(lay_5_W, a4) + lay_5_b\n",
    "a5 = tf.abs(z5) \n",
    "\n",
    "z6 = tf.matmul(lay_6_W, a5) + lay_6_b\n",
    "a6 = tf.abs(z6)\n",
    "\n",
    "z7 = tf.matmul(lay_7_W, a6) + lay_7_b\n",
    "a7 = tf.abs(z7)\n",
    "\n",
    "z8 = tf.matmul(lay_8_W, a7) + lay_8_b\n",
    "a8 = tf.abs(z8)\n",
    "\n",
    "z9 = tf.matmul(lay_9_W, a8) + lay_9_b\n",
    "a9 = tf.abs(z9)\n",
    "\n",
    "z10 = tf.matmul(lay_10_W, a9) + lay_10_b\n",
    "a10 = tf.abs(z10)\n",
    "\n",
    "z11 = tf.matmul(lay_11_W, a10) + lay_11_b\n",
    "a11 = tf.abs(z11)\n",
    "\n",
    "z12 = tf.matmul(lay_12_W, a11) + lay_12_b\n",
    "a12 = tf.abs(z12)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# define the backward computation:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# in the backward computations, there are no actiavtion functions\n",
    "y_in_back = a12\n",
    "\n",
    "a1_back = tf.abs(tf.matmul(tf.transpose(lay_12_W), y_in_back) + lay_11_b)\n",
    "a2_back = tf.abs(tf.matmul(tf.transpose(lay_11_W), a1_back) + lay_10_b)\n",
    "a3_back = tf.abs(tf.matmul(tf.transpose(lay_10_W), a2_back) + lay_9_b)\n",
    "a4_back = tf.abs(tf.matmul(tf.transpose(lay_9_W), a3_back) + lay_8_b)\n",
    "a5_back = tf.abs(tf.matmul(tf.transpose(lay_8_W), a4_back) + lay_7_b)\n",
    "a6_back = tf.abs(tf.matmul(tf.transpose(lay_7_W), a5_back) + lay_6_b)\n",
    "a7_back = tf.abs(tf.matmul(tf.transpose(lay_6_W), a6_back) + lay_5_b)\n",
    "a8_back = tf.abs(tf.matmul(tf.transpose(lay_5_W), a7_back) + lay_4_b)\n",
    "a9_back = tf.abs(tf.matmul(tf.transpose(lay_4_W), a8_back) + lay_3_b)\n",
    "a10_back = tf.abs(tf.matmul(tf.transpose(lay_3_W), a9_back) + lay_2_b)\n",
    "a11_back = tf.abs(tf.matmul(tf.transpose(lay_2_W), a10_back) + lay_1_b)\n",
    "a12_back = tf.abs(tf.matmul(tf.transpose(lay_1_W), a11_back) + lay_0_b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'Abs_11:0' shape=(10, ?) dtype=float32>"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_in_back"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "in_back_vector = tf.placeholder(tf.float32, shape=(num_classes, None))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# computations for obtaining predictions: \n",
    "pred1_back = tf.abs(tf.matmul(tf.transpose(lay_12_W), in_back_vector) + lay_11_b)\n",
    "pred2_back = tf.abs(tf.matmul(tf.transpose(lay_11_W), pred1_back) + lay_10_b)\n",
    "pred3_back = tf.abs(tf.matmul(tf.transpose(lay_10_W), pred2_back) + lay_9_b)\n",
    "pred4_back = tf.abs(tf.matmul(tf.transpose(lay_9_W), pred3_back) + lay_8_b)\n",
    "pred5_back = tf.abs(tf.matmul(tf.transpose(lay_8_W), pred4_back) + lay_7_b)\n",
    "pred6_back = tf.abs(tf.matmul(tf.transpose(lay_7_W), pred5_back) + lay_6_b)\n",
    "pred7_back = tf.abs(tf.matmul(tf.transpose(lay_6_W), pred6_back) + lay_5_b)\n",
    "pred8_back = tf.abs(tf.matmul(tf.transpose(lay_5_W), pred7_back) + lay_4_b)\n",
    "pred9_back = tf.abs(tf.matmul(tf.transpose(lay_4_W), pred8_back) + lay_3_b)\n",
    "pred10_back = tf.abs(tf.matmul(tf.transpose(lay_3_W), pred9_back) + lay_2_b)\n",
    "pred11_back = tf.abs(tf.matmul(tf.transpose(lay_2_W), pred10_back) + lay_1_b)\n",
    "pred12_back = tf.abs(tf.matmul(tf.transpose(lay_1_W), pred11_back) + lay_0_b)\n",
    "\n",
    "# generated digits:\n",
    "generated_digits = pred12_back"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(<tf.Tensor 'Abs_23:0' shape=(784, ?) dtype=float32>,\n",
       " <tf.Tensor 'Placeholder:0' shape=(784, ?) dtype=float32>)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_out_back = a12_back\n",
    "x_out_back, tf_input_pixels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_out_back_image = tf.reshape(tf.transpose(x_out_back), shape=(-1, 28, 28, 1))\n",
    "output_image_summary = tf.summary.image(\"output_image\", x_out_back_image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'Abs_11:0' shape=(10, ?) dtype=float32>"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_in_back"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Now compute the forward cost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def normalize(x):\n",
    "    '''\n",
    "        function to range normalize the given input tensor\n",
    "        @param \n",
    "        x => the input tensor to be range normalized\n",
    "        @return => range normalized tensor\n",
    "    '''\n",
    "    sqrs = tf.square(x)\n",
    "    x_mag = tf.sqrt(tf.reduce_sum(sqrs, axis=0, keep_dims=True))\n",
    "    # return the range normalized prediction values:\n",
    "    return (x / x_mag)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# forward cost \n",
    "fwd_cost = tf.reduce_mean(tf.abs(normalize(y_in_back) - tf_one_hot_encoded_labels))\n",
    "fwd_cost_summary = tf.summary.scalar(\"Forward_cost\", fwd_cost)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Now compute the backward cost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# backward cost \n",
    "# The backward cost is the mean squared error function\n",
    "bwd_cost = tf.reduce_mean(tf.abs(x_out_back - tf_input_pixels))\n",
    "bwd_cost_summary = tf.summary.scalar(\"Backward_cost\", bwd_cost)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# The final cost is the addition of both forward and the backward costs "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cost = fwd_cost + bwd_cost\n",
    "final_cost_summary = tf.summary.scalar(\"Final_cost\", cost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# define an optimizer for this task\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=lr).minimize(cost)\n",
    "init = tf.global_variables_initializer()\n",
    "all_summaries = tf.summary.merge_all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "n_train_examples = train_X.shape[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sess = tf.InteractiveSession()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tensorboard_writer = tf.summary.FileWriter(model_path_name, graph=sess.graph, filename_suffix=\".bot\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sess.run(init)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration: 0 Cost: 44.235\n",
      "Iteration: 100 Cost: 33.3513\n",
      "Iteration: 200 Cost: 34.5095\n",
      "Iteration: 300 Cost: 31.815\n",
      "Iteration: 400 Cost: 32.6158\n",
      "Iteration: 500 Cost: 30.4053\n",
      "Iteration: 600 Cost: 33.1326\n",
      "epoch no: 1\n",
      "Average epoch cost: 33.5398446811\n",
      "Iteration: 624 Cost: 33.3701\n",
      "Iteration: 724 Cost: 32.6056\n",
      "Iteration: 824 Cost: 34.6617\n",
      "Iteration: 924 Cost: 32.2532\n",
      "Iteration: 1024 Cost: 31.7901\n",
      "Iteration: 1124 Cost: 30.4964\n",
      "Iteration: 1224 Cost: 33.3142\n",
      "epoch no: 2\n",
      "Average epoch cost: 32.78271923\n",
      "Iteration: 1248 Cost: 33.0347\n",
      "Iteration: 1348 Cost: 31.3608\n",
      "Iteration: 1448 Cost: 33.4265\n",
      "Iteration: 1548 Cost: 31.3995\n",
      "Iteration: 1648 Cost: 32.6646\n",
      "Iteration: 1748 Cost: 29.6437\n",
      "Iteration: 1848 Cost: 32.8295\n",
      "epoch no: 3\n",
      "Average epoch cost: 32.6727665235\n",
      "Iteration: 1872 Cost: 33.6283\n",
      "Iteration: 1972 Cost: 31.8166\n",
      "Iteration: 2072 Cost: 33.5157\n",
      "Iteration: 2172 Cost: 31.193\n",
      "Iteration: 2272 Cost: 32.6616\n",
      "Iteration: 2372 Cost: 29.9695\n",
      "Iteration: 2472 Cost: 32.6124\n",
      "epoch no: 4\n",
      "Average epoch cost: 32.7223206728\n",
      "Iteration: 2496 Cost: 33.0209\n",
      "Iteration: 2596 Cost: 31.0569\n",
      "Iteration: 2696 Cost: 33.4844\n",
      "Iteration: 2796 Cost: 32.9818\n",
      "Iteration: 2896 Cost: 32.1798\n",
      "Iteration: 2996 Cost: 31.3818\n",
      "Iteration: 3096 Cost: 33.8631\n",
      "epoch no: 5\n",
      "Average epoch cost: 32.8299026978\n",
      "Iteration: 3120 Cost: 32.0413\n",
      "Iteration: 3220 Cost: 30.6998\n",
      "Iteration: 3320 Cost: 32.506\n",
      "Iteration: 3420 Cost: 31.3861\n",
      "Iteration: 3520 Cost: 32.2344\n",
      "Iteration: 3620 Cost: 30.6756\n",
      "Iteration: 3720 Cost: 31.5894\n",
      "epoch no: 6\n",
      "Average epoch cost: 32.8247696498\n",
      "Iteration: 3744 Cost: 33.0729\n",
      "Iteration: 3844 Cost: 32.3187\n",
      "Iteration: 3944 Cost: 33.6253\n",
      "Iteration: 4044 Cost: 32.5716\n",
      "Iteration: 4144 Cost: 31.7078\n",
      "Iteration: 4244 Cost: 31.1439\n",
      "Iteration: 4344 Cost: 32.2279\n",
      "epoch no: 7\n",
      "Average epoch cost: 32.8142951849\n",
      "Iteration: 4368 Cost: 33.6198\n",
      "Iteration: 4468 Cost: 32.2154\n",
      "Iteration: 4568 Cost: 33.5275\n",
      "Iteration: 4668 Cost: 33.88\n",
      "Iteration: 4768 Cost: 30.7947\n",
      "Iteration: 4868 Cost: 28.8094\n",
      "Iteration: 4968 Cost: 30.4041\n",
      "epoch no: 8\n",
      "Average epoch cost: 45.3173665145\n",
      "Iteration: 4992 Cost: 31.3984\n",
      "Iteration: 5092 Cost: 29.8607\n",
      "Iteration: 5192 Cost: 31.7613\n",
      "Iteration: 5292 Cost: 29.9562\n",
      "Iteration: 5392 Cost: 29.9054\n",
      "Iteration: 5492 Cost: 28.3563\n",
      "Iteration: 5592 Cost: 30.0934\n",
      "epoch no: 9\n",
      "Average epoch cost: 30.4936553637\n",
      "Iteration: 5616 Cost: 31.1904\n",
      "Iteration: 5716 Cost: 29.6475\n",
      "Iteration: 5816 Cost: 31.7387\n",
      "Iteration: 5916 Cost: 30.1402\n",
      "Iteration: 6016 Cost: 29.8389\n",
      "Iteration: 6116 Cost: 28.365\n",
      "Iteration: 6216 Cost: 30.0996\n",
      "epoch no: 10\n",
      "Average epoch cost: 30.3697260099\n",
      "Iteration: 6240 Cost: 31.0944\n",
      "Iteration: 6340 Cost: 29.6228\n",
      "Iteration: 6440 Cost: 31.6526\n",
      "Iteration: 6540 Cost: 29.7209\n",
      "Iteration: 6640 Cost: 29.7979\n",
      "Iteration: 6740 Cost: 28.2372\n",
      "Iteration: 6840 Cost: 29.975\n",
      "epoch no: 11\n",
      "Average epoch cost: 30.3381428902\n",
      "Iteration: 6864 Cost: 31.1643\n",
      "Iteration: 6964 Cost: 29.5064\n",
      "Iteration: 7064 Cost: 31.6402\n",
      "Iteration: 7164 Cost: 29.714\n",
      "Iteration: 7264 Cost: 29.6798\n",
      "Iteration: 7364 Cost: 28.2187\n",
      "Iteration: 7464 Cost: 29.8571\n",
      "epoch no: 12\n",
      "Average epoch cost: 30.2941971253\n",
      "Iteration: 7488 Cost: 31.2741\n",
      "Iteration: 7588 Cost: 29.6038\n",
      "Iteration: 7688 Cost: 31.8478\n",
      "Iteration: 7788 Cost: 29.7408\n",
      "Iteration: 7888 Cost: 35.0256\n",
      "Iteration: 7988 Cost: 31.0017\n",
      "Iteration: 8088 Cost: 30.7704\n",
      "epoch no: 13\n",
      "Average epoch cost: 31.3327684494\n",
      "Iteration: 8112 Cost: 31.7506\n",
      "Iteration: 8212 Cost: 29.8104\n",
      "Iteration: 8312 Cost: 33.0728\n",
      "Iteration: 8412 Cost: 30.1272\n",
      "Iteration: 8512 Cost: 29.8696\n",
      "Iteration: 8612 Cost: 28.2993\n",
      "Iteration: 8712 Cost: 30.2569\n",
      "epoch no: 14\n",
      "Average epoch cost: 31.0071069827\n",
      "Iteration: 8736 Cost: 31.1879\n",
      "Iteration: 8836 Cost: 29.5354\n",
      "Iteration: 8936 Cost: 33.8391\n",
      "Iteration: 9036 Cost: 32.913\n",
      "Iteration: 9136 Cost: 33.8256\n",
      "Iteration: 9236 Cost: 30.6716\n",
      "Iteration: 9336 Cost: 31.2045\n",
      "epoch no: 15\n",
      "Average epoch cost: 112.815323668\n",
      "Iteration: 9360 Cost: 32.2021\n",
      "Iteration: 9460 Cost: 30.4526\n",
      "Iteration: 9560 Cost: 32.0328\n",
      "Iteration: 9660 Cost: 30.0143\n",
      "Iteration: 9760 Cost: 30.1525\n",
      "Iteration: 9860 Cost: 28.5708\n",
      "Iteration: 9960 Cost: 30.2746\n",
      "epoch no: 16\n",
      "Average epoch cost: 30.7901429091\n",
      "Iteration: 9984 Cost: 31.375\n",
      "Iteration: 10084 Cost: 29.813\n",
      "Iteration: 10184 Cost: 31.8381\n",
      "Iteration: 10284 Cost: 29.9101\n",
      "Iteration: 10384 Cost: 29.9\n",
      "Iteration: 10484 Cost: 28.3994\n",
      "Iteration: 10584 Cost: 30.0862\n",
      "epoch no: 17\n",
      "Average epoch cost: 30.4972041509\n",
      "Iteration: 10608 Cost: 31.2523\n",
      "Iteration: 10708 Cost: 29.7199\n",
      "Iteration: 10808 Cost: 31.6847\n",
      "Iteration: 10908 Cost: 29.7986\n",
      "Iteration: 11008 Cost: 29.7818\n",
      "Iteration: 11108 Cost: 28.3173\n",
      "Iteration: 11208 Cost: 29.9989\n",
      "epoch no: 18\n",
      "Average epoch cost: 30.3953744234\n",
      "Iteration: 11232 Cost: 31.1702\n",
      "Iteration: 11332 Cost: 29.6387\n",
      "Iteration: 11432 Cost: 31.6568\n",
      "Iteration: 11532 Cost: 29.8689\n",
      "Iteration: 11632 Cost: 29.8463\n",
      "Iteration: 11732 Cost: 28.2608\n",
      "Iteration: 11832 Cost: 29.9344\n",
      "epoch no: 19\n",
      "Average epoch cost: 30.3446355783\n",
      "Iteration: 11856 Cost: 31.1504\n",
      "Iteration: 11956 Cost: 29.561\n",
      "Iteration: 12056 Cost: 31.8897\n",
      "Iteration: 12156 Cost: 29.79\n",
      "Iteration: 12256 Cost: 29.7883\n",
      "Iteration: 12356 Cost: 28.2447\n",
      "Iteration: 12456 Cost: 29.923\n",
      "epoch no: 20\n",
      "Average epoch cost: 30.3412335958\n",
      "Iteration: 12480 Cost: 31.0733\n",
      "Iteration: 12580 Cost: 29.5733\n",
      "Iteration: 12680 Cost: 31.6448\n",
      "Iteration: 12780 Cost: 29.8251\n",
      "Iteration: 12880 Cost: 29.6949\n",
      "Iteration: 12980 Cost: 28.2737\n",
      "Iteration: 13080 Cost: 30.1548\n",
      "epoch no: 21\n",
      "Average epoch cost: 30.3062419463\n",
      "Iteration: 13104 Cost: 31.0659\n",
      "Iteration: 13204 Cost: 29.5642\n",
      "Iteration: 13304 Cost: 31.599\n",
      "Iteration: 13404 Cost: 29.7228\n",
      "Iteration: 13504 Cost: 29.774\n",
      "Iteration: 13604 Cost: 28.2105\n",
      "Iteration: 13704 Cost: 29.9338\n",
      "epoch no: 22\n",
      "Average epoch cost: 30.2909487547\n",
      "Iteration: 13728 Cost: 31.1521\n",
      "Iteration: 13828 Cost: 29.5925\n",
      "Iteration: 13928 Cost: 31.6127\n",
      "Iteration: 14028 Cost: 30.9201\n",
      "Iteration: 14128 Cost: 30.0502\n",
      "Iteration: 14228 Cost: 28.5153\n",
      "Iteration: 14328 Cost: 29.9922\n",
      "epoch no: 23\n",
      "Average epoch cost: 30.4287522023\n",
      "Iteration: 14352 Cost: 31.2229\n",
      "Iteration: 14452 Cost: 29.6273\n",
      "Iteration: 14552 Cost: 31.565\n",
      "Iteration: 14652 Cost: 32.7717\n",
      "Iteration: 14752 Cost: 30.5288\n",
      "Iteration: 14852 Cost: 28.5546\n",
      "Iteration: 14952 Cost: 30.225\n",
      "epoch no: 24\n",
      "Average epoch cost: 30.9659326229\n",
      "Iteration: 14976 Cost: 31.2275\n",
      "Iteration: 15076 Cost: 29.6453\n",
      "Iteration: 15176 Cost: 32.603\n",
      "Iteration: 15276 Cost: 30.0602\n",
      "Iteration: 15376 Cost: 30.1112\n",
      "Iteration: 15476 Cost: 28.308\n",
      "Iteration: 15576 Cost: 30.1686\n",
      "epoch no: 25\n",
      "Average epoch cost: 30.6940831863\n",
      "Iteration: 15600 Cost: 31.107\n",
      "Iteration: 15700 Cost: 32.7236\n",
      "Iteration: 15800 Cost: 32.3147\n",
      "Iteration: 15900 Cost: 32.0413\n",
      "Iteration: 16000 Cost: 33.7636\n",
      "Iteration: 16100 Cost: 29.9733\n",
      "Iteration: 16200 Cost: 31.5448\n",
      "epoch no: 26\n",
      "Average epoch cost: 32.1627470224\n",
      "Iteration: 16224 Cost: 32.8564\n",
      "Iteration: 16324 Cost: 31.6252\n",
      "Iteration: 16424 Cost: 33.9283\n",
      "Iteration: 16524 Cost: 33.0116\n",
      "Iteration: 16624 Cost: 32.3723\n",
      "Iteration: 16724 Cost: 30.3784\n",
      "Iteration: 16824 Cost: 32.9227\n",
      "epoch no: 27\n",
      "Average epoch cost: 32.3803476615\n",
      "Iteration: 16848 Cost: 32.6697\n",
      "Iteration: 16948 Cost: 31.3957\n",
      "Iteration: 17048 Cost: 33.1894\n",
      "Iteration: 17148 Cost: 31.7284\n",
      "Iteration: 17248 Cost: 32.2109\n",
      "Iteration: 17348 Cost: 29.4915\n",
      "Iteration: 17448 Cost: 30.8825\n",
      "epoch no: 28\n",
      "Average epoch cost: 32.2304940315\n",
      "Iteration: 17472 Cost: 32.1273\n",
      "Iteration: 17572 Cost: 31.7374\n",
      "Iteration: 17672 Cost: 32.6512\n",
      "Iteration: 17772 Cost: 30.9016\n",
      "Iteration: 17872 Cost: 32.4211\n",
      "Iteration: 17972 Cost: 30.3369\n",
      "Iteration: 18072 Cost: 32.7316\n",
      "epoch no: 29\n",
      "Average epoch cost: 32.5244287803\n",
      "Iteration: 18096 Cost: 33.3709\n",
      "Iteration: 18196 Cost: 32.1702\n",
      "Iteration: 18296 Cost: 33.9655\n",
      "Iteration: 18396 Cost: 32.9505\n",
      "Iteration: 18496 Cost: 394.401\n",
      "Iteration: 18596 Cost: 29.3644\n",
      "Iteration: 18696 Cost: 31.0848\n",
      "epoch no: 30\n",
      "Average epoch cost: 104.310987583\n",
      "Iteration: 18720 Cost: 31.6703\n",
      "Iteration: 18820 Cost: 30.138\n",
      "Iteration: 18920 Cost: 31.8727\n",
      "Iteration: 19020 Cost: 29.937\n",
      "Iteration: 19120 Cost: 30.0729\n",
      "Iteration: 19220 Cost: 28.4562\n",
      "Iteration: 19320 Cost: 30.2036\n",
      "epoch no: 31\n",
      "Average epoch cost: 30.6637825018\n",
      "Iteration: 19344 Cost: 31.3599\n",
      "Iteration: 19444 Cost: 29.7296\n",
      "Iteration: 19544 Cost: 31.7817\n",
      "Iteration: 19644 Cost: 29.7887\n",
      "Iteration: 19744 Cost: 29.8239\n",
      "Iteration: 19844 Cost: 28.3535\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration: 19944 Cost: 30.04\n",
      "epoch no: 32\n",
      "Average epoch cost: 30.4628185767\n",
      "Iteration: 19968 Cost: 31.1525\n",
      "Iteration: 20068 Cost: 29.6038\n",
      "Iteration: 20168 Cost: 31.6672\n",
      "Iteration: 20268 Cost: 29.8759\n",
      "Iteration: 20368 Cost: 29.8044\n",
      "Iteration: 20468 Cost: 28.3499\n",
      "Iteration: 20568 Cost: 30.0102\n",
      "epoch no: 33\n",
      "Average epoch cost: 30.3756855787\n",
      "Iteration: 20592 Cost: 31.2122\n",
      "Iteration: 20692 Cost: 29.6015\n",
      "Iteration: 20792 Cost: 31.5972\n",
      "Iteration: 20892 Cost: 29.8218\n",
      "Iteration: 20992 Cost: 29.7289\n",
      "Iteration: 21092 Cost: 28.2955\n",
      "Iteration: 21192 Cost: 29.8975\n",
      "epoch no: 34\n",
      "Average epoch cost: 30.3118484173\n",
      "Iteration: 21216 Cost: 31.0809\n",
      "Iteration: 21316 Cost: 29.5648\n",
      "Iteration: 21416 Cost: 31.5527\n",
      "Iteration: 21516 Cost: 29.7283\n",
      "Iteration: 21616 Cost: 29.6932\n",
      "Iteration: 21716 Cost: 28.2138\n",
      "Iteration: 21816 Cost: 29.9234\n",
      "epoch no: 35\n",
      "Average epoch cost: 30.3053524066\n",
      "Iteration: 21840 Cost: 31.0638\n",
      "Iteration: 21940 Cost: 29.5496\n",
      "Iteration: 22040 Cost: 31.5525\n",
      "Iteration: 22140 Cost: 29.9392\n",
      "Iteration: 22240 Cost: 29.6753\n",
      "Iteration: 22340 Cost: 28.2429\n",
      "Iteration: 22440 Cost: 29.86\n",
      "epoch no: 36\n",
      "Average epoch cost: 30.2961833049\n",
      "Iteration: 22464 Cost: 31.0384\n",
      "Iteration: 22564 Cost: 29.5345\n",
      "Iteration: 22664 Cost: 31.5476\n",
      "Iteration: 22764 Cost: 29.686\n",
      "Iteration: 22864 Cost: 29.7788\n",
      "Iteration: 22964 Cost: 28.3615\n",
      "Iteration: 23064 Cost: 29.8794\n",
      "epoch no: 37\n",
      "Average epoch cost: 30.2576670861\n",
      "Iteration: 23088 Cost: 31.0567\n",
      "Iteration: 23188 Cost: 29.5139\n",
      "Iteration: 23288 Cost: 31.5223\n",
      "Iteration: 23388 Cost: 29.7963\n",
      "Iteration: 23488 Cost: 29.7326\n",
      "Iteration: 23588 Cost: 28.2296\n",
      "Iteration: 23688 Cost: 29.9297\n",
      "epoch no: 38\n",
      "Average epoch cost: 30.2873135041\n",
      "Iteration: 23712 Cost: 31.1181\n",
      "Iteration: 23812 Cost: 29.6438\n",
      "Iteration: 23912 Cost: 33.0197\n",
      "Iteration: 24012 Cost: 30.9709\n",
      "Iteration: 24112 Cost: 30.5678\n",
      "Iteration: 24212 Cost: 28.9356\n",
      "Iteration: 24312 Cost: 30.4872\n",
      "epoch no: 39\n",
      "Average epoch cost: 31.1167053718\n",
      "Iteration: 24336 Cost: 31.472\n",
      "Iteration: 24436 Cost: 30.7177\n",
      "Iteration: 24536 Cost: 33.5358\n",
      "Iteration: 24636 Cost: 30.9001\n",
      "Iteration: 24736 Cost: 30.1758\n",
      "Iteration: 24836 Cost: 28.2872\n",
      "Iteration: 24936 Cost: 33.361\n",
      "epoch no: 40\n",
      "Average epoch cost: 31.4157378704\n",
      "Iteration: 24960 Cost: 35.1674\n",
      "Iteration: 25060 Cost: 30.827\n",
      "Iteration: 25160 Cost: 32.4524\n",
      "Iteration: 25260 Cost: 30.7653\n",
      "Iteration: 25360 Cost: 31.6068\n",
      "Iteration: 25460 Cost: 3325.18\n",
      "Iteration: 25560 Cost: 34.5199\n",
      "epoch no: 41\n",
      "Average epoch cost: 776.671597955\n",
      "Iteration: 25584 Cost: 34.5778\n",
      "Iteration: 25684 Cost: 33.3209\n",
      "Iteration: 25784 Cost: 33.9466\n",
      "Iteration: 25884 Cost: 31.2339\n",
      "Iteration: 25984 Cost: 30.9329\n",
      "Iteration: 26084 Cost: 29.2181\n",
      "Iteration: 26184 Cost: 30.8389\n",
      "epoch no: 42\n",
      "Average epoch cost: 32.3013492272\n",
      "Iteration: 26208 Cost: 31.6977\n",
      "Iteration: 26308 Cost: 30.4006\n",
      "Iteration: 26408 Cost: 32.249\n",
      "Iteration: 26508 Cost: 30.4144\n",
      "Iteration: 26608 Cost: 30.3155\n",
      "Iteration: 26708 Cost: 28.679\n",
      "Iteration: 26808 Cost: 30.668\n",
      "epoch no: 43\n",
      "Average epoch cost: 30.9519140782\n",
      "Iteration: 26832 Cost: 31.5436\n",
      "Iteration: 26932 Cost: 30.4694\n",
      "Iteration: 27032 Cost: 32.1378\n",
      "Iteration: 27132 Cost: 30.3725\n",
      "Iteration: 27232 Cost: 30.2053\n",
      "Iteration: 27332 Cost: 28.9452\n",
      "Iteration: 27432 Cost: 30.5461\n",
      "epoch no: 44\n",
      "Average epoch cost: 30.8321855129\n",
      "Iteration: 27456 Cost: 31.4957\n",
      "Iteration: 27556 Cost: 30.2851\n",
      "Iteration: 27656 Cost: 31.9816\n",
      "Iteration: 27756 Cost: 30.0829\n",
      "Iteration: 27856 Cost: 30.316\n",
      "Iteration: 27956 Cost: 28.7219\n",
      "Iteration: 28056 Cost: 30.4345\n",
      "epoch no: 45\n",
      "Average epoch cost: 30.7469712526\n",
      "Iteration: 28080 Cost: 31.5499\n",
      "Iteration: 28180 Cost: 29.9774\n",
      "Iteration: 28280 Cost: 31.9456\n",
      "Iteration: 28380 Cost: 29.9672\n",
      "Iteration: 28480 Cost: 30.1022\n",
      "Iteration: 28580 Cost: 28.5281\n",
      "Iteration: 28680 Cost: 30.2541\n",
      "epoch no: 46\n",
      "Average epoch cost: 30.6852237536\n",
      "Iteration: 28704 Cost: 31.4461\n",
      "Iteration: 28804 Cost: 29.8494\n",
      "Iteration: 28904 Cost: 31.9349\n",
      "Iteration: 29004 Cost: 30.1636\n",
      "Iteration: 29104 Cost: 30.0414\n",
      "Iteration: 29204 Cost: 28.5845\n",
      "Iteration: 29304 Cost: 30.2995\n",
      "epoch no: 47\n",
      "Average epoch cost: 30.6083538349\n",
      "Iteration: 29328 Cost: 31.3995\n",
      "Iteration: 29428 Cost: 29.888\n",
      "Iteration: 29528 Cost: 31.7885\n",
      "Iteration: 29628 Cost: 29.912\n",
      "Iteration: 29728 Cost: 29.9231\n",
      "Iteration: 29828 Cost: 28.4973\n",
      "Iteration: 29928 Cost: 30.2499\n",
      "epoch no: 48\n",
      "Average epoch cost: 30.5491834909\n",
      "Iteration: 29952 Cost: 31.368\n",
      "Iteration: 30052 Cost: 29.7436\n",
      "Iteration: 30152 Cost: 31.8945\n",
      "Iteration: 30252 Cost: 29.8214\n",
      "Iteration: 30352 Cost: 29.8594\n",
      "Iteration: 30452 Cost: 28.4157\n",
      "Iteration: 30552 Cost: 30.1269\n",
      "epoch no: 49\n",
      "Average epoch cost: 30.5071100272\n",
      "Iteration: 30576 Cost: 31.2072\n",
      "Iteration: 30676 Cost: 29.7366\n",
      "Iteration: 30776 Cost: 31.821\n",
      "Iteration: 30876 Cost: 29.9203\n",
      "Iteration: 30976 Cost: 29.8293\n",
      "Iteration: 31076 Cost: 28.4773\n",
      "Iteration: 31176 Cost: 30.1105\n",
      "epoch no: 50\n",
      "Average epoch cost: 30.4628252341\n",
      "Iteration: 31200 Cost: 31.3759\n",
      "Iteration: 31300 Cost: 29.9475\n",
      "Iteration: 31400 Cost: 31.736\n",
      "Iteration: 31500 Cost: 29.8409\n",
      "Iteration: 31600 Cost: 29.8916\n",
      "Iteration: 31700 Cost: 28.4099\n",
      "Iteration: 31800 Cost: 30.1706\n",
      "epoch no: 51\n",
      "Average epoch cost: 30.4582342246\n",
      "Iteration: 31824 Cost: 31.3474\n",
      "Iteration: 31924 Cost: 29.8287\n",
      "Iteration: 32024 Cost: 31.7995\n",
      "Iteration: 32124 Cost: 29.934\n",
      "Iteration: 32224 Cost: 29.8499\n",
      "Iteration: 32324 Cost: 28.4457\n",
      "Iteration: 32424 Cost: 30.1827\n",
      "epoch no: 52\n",
      "Average epoch cost: 30.494361856\n",
      "Iteration: 32448 Cost: 31.3124\n",
      "Iteration: 32548 Cost: 29.7206\n",
      "Iteration: 32648 Cost: 31.8126\n",
      "Iteration: 32748 Cost: 29.8848\n",
      "Iteration: 32848 Cost: 29.9097\n",
      "Iteration: 32948 Cost: 28.34\n",
      "Iteration: 33048 Cost: 30.1299\n",
      "epoch no: 53\n",
      "Average epoch cost: 30.4621426264\n",
      "Iteration: 33072 Cost: 31.4116\n",
      "Iteration: 33172 Cost: 29.6962\n",
      "Iteration: 33272 Cost: 31.9074\n",
      "Iteration: 33372 Cost: 29.8489\n",
      "Iteration: 33472 Cost: 30.2636\n",
      "Iteration: 33572 Cost: 28.4885\n",
      "Iteration: 33672 Cost: 30.1704\n",
      "epoch no: 54\n",
      "Average epoch cost: 30.4716954109\n",
      "Iteration: 33696 Cost: 31.2876\n",
      "Iteration: 33796 Cost: 29.8345\n",
      "Iteration: 33896 Cost: 33.4428\n",
      "Iteration: 33996 Cost: 459.239\n",
      "Iteration: 34096 Cost: 36.2695\n",
      "Iteration: 34196 Cost: 33.248\n",
      "Iteration: 34296 Cost: 35.4311\n",
      "epoch no: 55\n",
      "Average epoch cost: 4368.28539341\n",
      "Iteration: 34320 Cost: 35.449\n",
      "Iteration: 34420 Cost: 33.8533\n",
      "Iteration: 34520 Cost: 34.96\n",
      "Iteration: 34620 Cost: 33.0936\n",
      "Iteration: 34720 Cost: 33.0732\n",
      "Iteration: 34820 Cost: 31.1062\n",
      "Iteration: 34920 Cost: 33.4959\n",
      "epoch no: 56\n",
      "Average epoch cost: 33.8456985492\n",
      "Iteration: 34944 Cost: 34.32\n",
      "Iteration: 35044 Cost: 32.6428\n",
      "Iteration: 35144 Cost: 34.1116\n",
      "Iteration: 35244 Cost: 32.0216\n",
      "Iteration: 35344 Cost: 32.1065\n",
      "Iteration: 35444 Cost: 30.4821\n",
      "Iteration: 35544 Cost: 32.2664\n",
      "epoch no: 57\n",
      "Average epoch cost: 32.7910116544\n",
      "Iteration: 35568 Cost: 33.1087\n",
      "Iteration: 35668 Cost: 31.81\n",
      "Iteration: 35768 Cost: 33.7004\n",
      "Iteration: 35868 Cost: 31.5139\n",
      "Iteration: 35968 Cost: 31.6501\n",
      "Iteration: 36068 Cost: 29.912\n",
      "Iteration: 36168 Cost: 31.7838\n",
      "epoch no: 58\n",
      "Average epoch cost: 32.2208949083\n",
      "Iteration: 36192 Cost: 32.6887\n",
      "Iteration: 36292 Cost: 31.3068\n",
      "Iteration: 36392 Cost: 33.1069\n",
      "Iteration: 36492 Cost: 31.1682\n",
      "Iteration: 36592 Cost: 31.473\n",
      "Iteration: 36692 Cost: 29.5646\n",
      "Iteration: 36792 Cost: 31.5885\n",
      "epoch no: 59\n",
      "Average epoch cost: 31.8411430304\n",
      "Iteration: 36816 Cost: 32.5747\n",
      "Iteration: 36916 Cost: 31.2436\n",
      "Iteration: 37016 Cost: 33.0004\n",
      "Iteration: 37116 Cost: 30.8872\n",
      "Iteration: 37216 Cost: 31.0824\n",
      "Iteration: 37316 Cost: 29.3478\n",
      "Iteration: 37416 Cost: 31.6561\n",
      "epoch no: 60\n",
      "Average epoch cost: 31.6535185484\n",
      "Iteration: 37440 Cost: 32.432\n",
      "Iteration: 37540 Cost: 30.9554\n",
      "Iteration: 37640 Cost: 32.7566\n",
      "Iteration: 37740 Cost: 30.8771\n",
      "Iteration: 37840 Cost: 30.9417\n",
      "Iteration: 37940 Cost: 29.4081\n",
      "Iteration: 38040 Cost: 31.1347\n",
      "epoch no: 61\n",
      "Average epoch cost: 31.5093403749\n",
      "Iteration: 38064 Cost: 32.386\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-44-ca8dd28c72f4>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     12\u001b[0m         \u001b[0;31m# run the computation:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m         _, loss = sess.run((optimizer, cost), feed_dict={tf_input_pixels: train_X_minibatch, \n\u001b[0;32m---> 14\u001b[0;31m                                                          tf_integer_labels: train_Y_minibatch})\n\u001b[0m\u001b[1;32m     15\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m         \u001b[0;31m# add the cost to the cost list\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/animesh/Programming/platforms/anaconda3/envs/snakes/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    887\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    888\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 889\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    890\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    891\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/animesh/Programming/platforms/anaconda3/envs/snakes/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1118\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1119\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1120\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1121\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1122\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/animesh/Programming/platforms/anaconda3/envs/snakes/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1315\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1316\u001b[0m       return self._do_call(_run_fn, self._session, feeds, fetches, targets,\n\u001b[0;32m-> 1317\u001b[0;31m                            options, run_metadata)\n\u001b[0m\u001b[1;32m   1318\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1319\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/animesh/Programming/platforms/anaconda3/envs/snakes/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1321\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1322\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1323\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1324\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1325\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/animesh/Programming/platforms/anaconda3/envs/snakes/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1300\u001b[0m           return tf_session.TF_Run(session, options,\n\u001b[1;32m   1301\u001b[0m                                    \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1302\u001b[0;31m                                    status, run_metadata)\n\u001b[0m\u001b[1;32m   1303\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1304\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# start training the network for num_iterations and using the batch_size\n",
    "global_step = 0\n",
    "for epoch in range(no_of_epochs):\n",
    "    global_index = 0; costs = [] # start with empty list\n",
    "    while(global_index < n_train_examples):\n",
    "        start = global_index; end = start + batch_size\n",
    "        train_X_minibatch = train_X[:, start: end]\n",
    "        train_Y_minibatch = train_Y.astype(np.int32)[start: end]\n",
    "\n",
    "        iteration = global_index / batch_size\n",
    "        \n",
    "        # run the computation:\n",
    "        _, loss = sess.run((optimizer, cost), feed_dict={tf_input_pixels: train_X_minibatch, \n",
    "                                                         tf_integer_labels: train_Y_minibatch})\n",
    "\n",
    "        # add the cost to the cost list\n",
    "        costs.append(loss)\n",
    "\n",
    "        if(iteration % 100 == 0):\n",
    "            sums = sess.run(all_summaries, feed_dict={tf_input_pixels: train_X_minibatch, \n",
    "                                                         tf_integer_labels: train_Y_minibatch})\n",
    "            \n",
    "            print \"Iteration: \" + str(global_step) + \" Cost: \" + str(loss)\n",
    "\n",
    "            tensorboard_writer.add_summary(sums, global_step = global_step)\n",
    "        \n",
    "        # increment the global index \n",
    "        global_index = global_index + batch_size\n",
    "    \n",
    "        global_step += 1\n",
    "        \n",
    "    # print the average epoch cost:\n",
    "    print \"epoch no: \" + str(epoch + 1)\n",
    "    print \"Average epoch cost: \" + str(sum(costs) / len(costs))\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Very Important: Save this trained model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_file_name = os.path.join(model_path_name, model_path_name.split(\"/\")[-1])\n",
    "model_file_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "saver.save(sess, model_file_name, global_step=global_step)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model13 seems to be the most promising till now. It has (Mean absolute difference) function as the forward and the backward costs\n",
    "-------------------------------------------------------------------------------------------------------------------\n",
    "# Model2_v2 (currently being used), is same as Model13 but with the softmax function replaced by the range normalizer function for getting a probability distribution"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# The following cell shows how the network final activations look like upon passing some of the test images that It has never seen before\n",
    "-------------------------------------------------------------------------------------------------------------------\n",
    "# Run the following cell multiple times to see the effect better"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "saver.restore(sess, tf.train.latest_checkpoint(model_path_name))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# check by plotting some image\n",
    "random_index = np.random.randint(test_X.shape[-1])\n",
    "random_image = test_X[:, random_index].reshape((28, 28))\n",
    "# use plt to plot the image\n",
    "plt.figure().suptitle(\"Label of the image: \" + str(test_Y[random_index]))\n",
    "plt.imshow(random_image)\n",
    "\n",
    "# generate the predictions for one random image from the test set.\n",
    "predictions = np.squeeze(sess.run(y_in_back, feed_dict={tf_input_pixels: test_X[:, random_index].reshape((-1, 1))}))\n",
    "\n",
    "plt.figure().suptitle(\"Predictions obtained from the network\")\n",
    "plt.plot(range(10), predictions);\n",
    "print predictions\n",
    "print \"Predicted label: \" + str(np.argmax(predictions))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Calculate the accuracy of the network on the training and the test dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf_input_pixels, train_X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "preds = sess.run(y_in_back, feed_dict={tf_input_pixels: train_X})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "correct = np.sum(np.argmax(preds, axis=0) == train_Y)\n",
    "accuracy = (float(correct) / train_X.shape[-1]) * 100\n",
    "print \"Training accuracy: \" + str(accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print \"No. of examples that the network got wrong: \" + str(n_train_examples - correct)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Error Analysis:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The following cell saves all the incorrectly classified images into the specified directory."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "pred_labels = np.argmax(preds, axis=0)\n",
    "for index in range(n_train_examples):\n",
    "    # check if the predicted label is equal to the ideal label\n",
    "    if(pred_labels[index] != train_Y[index]):\n",
    "        image = train_X[:, index].reshape((28, 28))\n",
    "        \n",
    "        pred_label = str(pred_labels[index])\n",
    "        orig_label = str(train_Y[index])\n",
    "        \n",
    "        # create a new image\n",
    "        plt.figure(index).suptitle(\"predicted label: \" + pred_label + \" original label: \" + orig_label)\n",
    "        plt.subplot(211)\n",
    "        plt.plot(preds[:, index])\n",
    "        plt.subplot(212)\n",
    "        plt.imshow(image)\n",
    "        plt.savefig(os.path.join(error_analysis_path, str(index)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Calculate the test accuracy:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_preds = sess.run(y_in_back, feed_dict={tf_input_pixels: test_X})\n",
    "test_correct = np.sum(np.argmax(test_preds, axis=0) == test_Y)\n",
    "test_accuracy = (float(test_correct) / test_X.shape[-1]) * 100\n",
    "print \"Testing accuracy:\" + str(test_accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Now comes the best part! Can The network generate digits?\n",
    "-------------------------------------------------------------------------------------------------------------------\n",
    "# Let's generate some digits by tweaking the learned representation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "generator_array = np.array([2.42562220e-03,   6.63316175e-02,   5.14896587e-03,   5.55557422e-02,\n",
    "   8.33549351e-03,   1.06856683e-02,   1.17659550e+01,   1.30331926e-02,\n",
    "   1.42113212e-02,   1.96716078e-02]).reshape(-1, 1).astype(np.float32)\n",
    "generator_array.dtype\n",
    "generated_image = sess.run(generated_digits, feed_dict={in_back_vector: generator_array}).reshape((28, 28))\n",
    "plt.imshow(generated_image)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# generate representations for different digits by walking along their respective axes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "total_frames = 80"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "all_digits = [] # start with an empty list\n",
    "for walking_axis in range(num_classes):\n",
    "    reps = np.zeros(shape=(num_classes, total_frames))\n",
    "    for cnt in range(total_frames):\n",
    "        reps[walking_axis, cnt] = cnt\n",
    "    all_digits.append(reps)\n",
    "\n",
    "all_digits = np.hstack(all_digits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_digits.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# obtain the images for these inputs:\n",
    "representations, images = sess.run((pred1_back, generated_digits), feed_dict={in_back_vector: all_digits})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_digits = all_digits.T\n",
    "print all_digits.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "representations = representations.T\n",
    "print representations.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "images = images.T.reshape((-1, 28, 28))\n",
    "print images.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "imagelist = images\n",
    "\n",
    "import matplotlib.animation as animation\n",
    "from IPython.display import HTML\n",
    "plt.rcParams['animation.ffmpeg_path'] = u'/home/animesh/.linuxbrew/bin/ffmpeg'\n",
    "\n",
    "fig = plt.figure() # make figure\n",
    "\n",
    "# make axesimage object\n",
    "# the vmin and vmax here are very important to get the color map correct\n",
    "im = plt.imshow(imagelist[0], cmap=plt.get_cmap('jet'), vmin=0, vmax=1);\n",
    "\n",
    "# function to update figure\n",
    "def updatefig(j):\n",
    "    # set the data in the axesimage object\n",
    "    im.set_array(imagelist[j])\n",
    "    # return the artists set\n",
    "    return [im]\n",
    "# kick off the animation\n",
    "ani = animation.FuncAnimation(fig, updatefig, frames=range(images.shape[0]), \n",
    "                              interval=50, blit=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "in_list = all_digits\n",
    "\n",
    "fig = plt.figure() # make figure\n",
    "\n",
    "# make axesimage object\n",
    "# the vmin and vmax here are very important to get the color map correct\n",
    "plt.ylim((0, 80))\n",
    "in_plot,  = plt.plot(in_list[0]);\n",
    "\n",
    "# function to update figure\n",
    "def updatefig(j):\n",
    "    # set the data in the axesimage object\n",
    "    in_plot.set_ydata(in_list[j])\n",
    "    # return the artists set\n",
    "    return [in_plot]\n",
    "# kick off the animation\n",
    "in_ani = animation.FuncAnimation(fig, updatefig, frames=range(all_digits.shape[0]), \n",
    "                              interval=50, blit=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plotlist = representations\n",
    "\n",
    "fig = plt.figure() # make figure\n",
    "\n",
    "# make axesimage object\n",
    "# the vmin and vmax here are very important to get the color map correct\n",
    "plt.ylim((0, 0.8))\n",
    "plot,  = plt.plot(representations[0]);\n",
    "\n",
    "# function to update figure\n",
    "def updatefig(j):\n",
    "    # set the data in the axesimage object\n",
    "    plot.set_ydata(representations[j])\n",
    "    # return the artists set\n",
    "    return [plot]\n",
    "# kick off the animation\n",
    "plot_ani = animation.FuncAnimation(fig, updatefig, frames=range(representations.shape[0]), \n",
    "                              interval=50, blit=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(animation.writers.list())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "HTML(in_ani.to_html5_video())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "HTML(ani.to_html5_video())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "HTML(plot_ani.to_html5_video())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# The above activations' graph visualization proves that the network is not cheating by using half of the representations for encoding and half for decoding. The network has simultaneously stored weights that allows it to perform both the tasks in a unified manner."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hell Yeah! That's a victory! We can indeed generate digits using this neural network"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}