DNN_NumPy.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<pre><h1>E1-313 TIPR Assignment 2 Code base & Report</h1>\n",
    "<h2>Neural Network Implementation in Python3</h2>\n",
    "<h3><i> - Achint Chaudhary</i></h3>\n",
    "<h3>15879, M.Tech (CSA)</h3>\n",
    "<h5>Note:</h5> Please Scroll Down for Report section, or search \"Part 1\"\n",
    "<img src=\"Images/dnn_architecture.png\"/>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<pre>\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Standard Library Imports</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys, os, shutil, itertools as it\n",
    "from copy import deepcopy\n",
    "from datetime import datetime\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from scipy import ndimage\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from skimage import io\n",
    "from sklearn.metrics import f1_score, accuracy_score, classification_report\n",
    "\n",
    "import keras\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout\n",
    "from keras.optimizers import Adam\n",
    "from keras.callbacks import EarlyStopping\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"error\")\n",
    "\n",
    "try:\n",
    "    res_stdout\n",
    "except:\n",
    "    res_stdout = (sys.stdout if sys.stdout else sys.__stdout__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "verbose = bool( eval(input('Do you want Verbose??: 0/1  ')))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Changing File I/O & Matplotlib inlining if not verbose</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "if not verbose:\n",
    "    sys.stdout = sys.__stdout__ = open('stdoutbuffer','a',buffering=1)\n",
    "    mpl.use('Agg')\n",
    "else:\n",
    "    sys.stdout = sys.__stdout__ = res_stdout\n",
    "    %matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<pre>\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Activation Functions</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "class ActV:\n",
    "    def sigmoid(x):\n",
    "        return 1/(1+np.exp(-x))\n",
    "    def relu(x):\n",
    "        return np.maximum(0,x)\n",
    "    def tanh(x):\n",
    "        return 2*ActV.sigmoid(x)-1\n",
    "    def swish(x):\n",
    "        return x*ActV.sigmoid(x)\n",
    "    def softmax(x):\n",
    "        x = x-x.max(axis=1,keepdims=True)\n",
    "        _ = np.exp(x)\n",
    "        return _/np.sum(_,axis=1,keepdims=True)\n",
    "\n",
    "class ActD:\n",
    "    def sigmoid(x):\n",
    "        _ = ActV.sigmoid( x )\n",
    "        return _ * (1-_)\n",
    "    def relu(x):\n",
    "        '1 for x>=0'\n",
    "        return (np.sign(x)>=0)\n",
    "    def tanh(x):\n",
    "        return 1-(ActV.tanh(x))**2\n",
    "    def swish(x):\n",
    "        'y’ = y + σ(x) . (1 – y)'\n",
    "        _1 = ActV.swish(x)\n",
    "        _2 = ActV.sigmoid(x)\n",
    "        return _1 + _2*(1-_1)\n",
    "    def softmax(x):# Still in doubt, it should be a matrix\n",
    "        _ = ActV.softmax( x )\n",
    "        return _ * (1-_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Adding \"Swish\" function to Keras</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Ref: https://stackoverflow.com/questions/43915482/how-do-you-create-a-custom-activation-function-with-keras\n",
    "from keras.layers import Activation\n",
    "from keras import backend as K\n",
    "from keras.utils.generic_utils import get_custom_objects\n",
    "\n",
    "def swish2(x):\n",
    "    return x*K.sigmoid(x)\n",
    "\n",
    "get_custom_objects().update({'swish': Activation(swish2)})\n",
    "\n",
    "def addswish(model):\n",
    "    model.add(Activation(swish2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Cost Functions & Performance Metrics</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "class CostV:\n",
    "    def cross_entropy(act, pred):\n",
    "        pred = np.where(act!=1,pred+np.e,pred) # Handling perfect prediction\n",
    "        pred = np.where(np.logical_and(act==1,pred==0),pred+10**-8,pred) # Handling imperfect prediction\n",
    "        return -1*np.mean( act*np.log(pred) ,axis=0,keepdims=True)\n",
    "    def MSE(act, pred):\n",
    "        return np.mean( (pred-act)**2 ,axis=0,keepdims=True)\n",
    "    \n",
    "class CostD:\n",
    "    def cross_entropy(act, pred):\n",
    "        return pred - act\n",
    "    def MSE(act, pred):\n",
    "        return 2*(pred-act)\n",
    "\n",
    "class Metrices:\n",
    "    def accuracy(act, pred):\n",
    "            return np.mean((act==pred).all(axis=1))\n",
    "\n",
    "def one_hot(y):\n",
    "    return 1*(y==y.max(axis=1,keepdims=True))\n",
    "\n",
    "def cattooht(Y):\n",
    "    Y = np.ravel(Y)\n",
    "    _ = sorted(set(Y))\n",
    "    tmp = np.zeros((Y.shape[0],len(_)),dtype='int32')\n",
    "    for i in range(len(Y)):\n",
    "        tmp[i][_.index(Y[i])] = 1\n",
    "    return tmp,_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Xavier-He Initialization</h3>\n",
    "<img src=\"Images/XHE.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def initWB(IP,OP,function='relu',He=True,mode='gaussian'):\n",
    "    if He:\n",
    "        # Xavier & He initialization\n",
    "        _ = 1/(IP+OP)**0.5\n",
    "        if function in ('sigmoid','softmax'):\n",
    "            r, s = 6**0.5, 2**0.5\n",
    "        elif function=='tanh':\n",
    "            r, s = 4*6**0.5, 4*2**0.5\n",
    "        else: # relu or swish function\n",
    "            r, s = 12**0.5, 2\n",
    "        r, s = r*_, s*_\n",
    "    else:\n",
    "        r, s = 1, 1\n",
    "    # Generating matrices\n",
    "    if mode=='uniform':\n",
    "        return 2*r*np.random.random((IP,OP))-r , 2*r*np.random.random((1,OP))-r\n",
    "    elif mode=='gaussian':\n",
    "        return np.random.randn(IP,OP)*s , np.random.randn(1,OP)*s\n",
    "    else:\n",
    "        raise Exception('Code should be unreachable')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Data split function family</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def RSplit(X,Y,K=10):\n",
    "    'Random Split Function'\n",
    "    _ = list(range(X.shape[0]))\n",
    "    index_set = []\n",
    "    indxs = set(_)\n",
    "    batch_size = round(X.shape[0]/K)\n",
    "    np.random.shuffle(_)\n",
    "    for k in range(0,X.shape[0],batch_size):\n",
    "        test = set(_[k:k+batch_size])\n",
    "        train = indxs - test\n",
    "        index_set.append((list(train),list(test)))\n",
    "    return index_set\n",
    "\n",
    "def SSplit(X,Y,K=10,seed=True):\n",
    "    'Stratified Split Function'\n",
    "    if seed:\n",
    "        np.random.seed(42)\n",
    "    Y = pd.DataFrame([tuple(y) for y in Y])\n",
    "    classes = set(Y)\n",
    "    c2i = {}\n",
    "    for index,label in Y.iterrows():\n",
    "        label = label[0]\n",
    "        if label in c2i:\n",
    "            c2i[label].add(index)\n",
    "        else:\n",
    "            c2i[label] = {index}\n",
    "    \n",
    "    # Each class -> list of indices\n",
    "    for i in c2i:\n",
    "        c2i[i] = list(c2i[i])\n",
    "        np.random.shuffle(c2i[i])\n",
    "    \n",
    "    # Each class with its set of train, test split indices\n",
    "    c2is = {}\n",
    "    for cls in c2i:\n",
    "        a = int(np.round(len(c2i[cls])/K))\n",
    "        c2is[cls] = []\n",
    "        for fold in range(K):\n",
    "            test_indices  = c2i[cls][a*fold:a*(fold+1)]\n",
    "            train_indices = c2i[cls][0:a*fold] + c2i[cls][a*(fold+1):]\n",
    "            c2is[cls].append((train_indices,test_indices))\n",
    "        np.random.shuffle(c2is[cls])\n",
    "        \n",
    "    index_set = []\n",
    "    for i in range(K):\n",
    "        train,test = set(),set()\n",
    "        for cls in c2is:\n",
    "            _ = c2is[cls][i]\n",
    "            train.update(set(_[0]))\n",
    "            test.update (set(_[1]))\n",
    "        index_set.append((list(train),list(test)))\n",
    "    return index_set\n",
    "\n",
    "def BSplit(X,Y,K=10):\n",
    "    'Biased Split Function'\n",
    "    indx = sorted(np.arange(X.shape[0]),key = lambda i:list(Y[i]))\n",
    "    indices = set(indx)\n",
    "    index_set = []\n",
    "    step = int(np.ceil(len(indx)/K))\n",
    "    for i in range(0,len(indx),step):\n",
    "        test = set(indx[i:i+step])\n",
    "        train = indices - test\n",
    "        index_set.append((list(train),list(test)))\n",
    "    return index_set\n",
    "\n",
    "def Split(X,Y,K=10,mode='R'):\n",
    "    if mode=='S':\n",
    "        return SSplit(X,Y,K)\n",
    "    elif mode=='B':\n",
    "        return BSplit(X,Y,K)\n",
    "    else:\n",
    "        return RSplit(X,Y,K)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Max-Pooling Code for Image Compression</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Ref: https://stackoverflow.com/questions/42463172/how-to-perform-max-mean-pooling-on-a-2d-array-using-numpy\n",
    "def asStride(arr,sub_shape,stride):\n",
    "    '''Get a strided sub-matrices view of an ndarray.\n",
    "    See also skimage.util.shape.view_as_windows()\n",
    "    '''\n",
    "    s0,s1 = arr.strides[:2]\n",
    "    m1,n1 = arr.shape[:2]\n",
    "    m2,n2 = sub_shape\n",
    "    view_shape = (1+(m1-m2)//stride[0],1+(n1-n2)//stride[1],m2,n2)+arr.shape[2:]\n",
    "    strides = (stride[0]*s0,stride[1]*s1,s0,s1)+arr.strides[2:]\n",
    "    subs = np.lib.stride_tricks.as_strided(arr,view_shape,strides=strides)\n",
    "    return subs\n",
    "\n",
    "def poolingOverlap(mat,ksize,stride=None,method='max',pad=False):\n",
    "    '''Overlapping pooling on 2D or 3D data.\n",
    "\n",
    "    <mat>: ndarray, input array to pool.\n",
    "    <ksize>: tuple of 2, kernel size in (ky, kx).\n",
    "    <stride>: tuple of 2 or None, stride of pooling window.\n",
    "              If None, same as <ksize> (non-overlapping pooling).\n",
    "    <method>: str, 'max for max-pooling,\n",
    "                   'mean' for mean-pooling.\n",
    "    <pad>: bool, pad <mat> or not. If no pad, output has size\n",
    "           (n-f)//s+1, n being <mat> size, f being kernel size, s stride.\n",
    "           if pad, output has size ceil(n/s).\n",
    "\n",
    "    Return <result>: pooled matrix.\n",
    "    '''\n",
    "    m, n = mat.shape[:2]\n",
    "    ky,kx = ksize\n",
    "    if stride is None:\n",
    "        stride = (ky,kx)\n",
    "    sy,sx = stride\n",
    "\n",
    "    _ceil = lambda x,y: int(np.ceil(x/float(y)))\n",
    "\n",
    "    if pad:\n",
    "        ny = _ceil(m,sy)\n",
    "        nx = _ceil(n,sx)\n",
    "        size = ((ny-1)*sy+ky, (nx-1)*sx+kx) + mat.shape[2:]\n",
    "        mat_pad = np.full(size,np.nan)\n",
    "        mat_pad[:m,:n,...]=mat\n",
    "    else:\n",
    "        mat_pad=mat[:(m-ky)//sy*sy+ky, :(n-kx)//sx*sx+kx, ...]\n",
    "\n",
    "    view=asStride(mat_pad,ksize,stride)\n",
    "\n",
    "    if method=='max':\n",
    "        result=np.nanmax(view,axis=(2,3))\n",
    "    else:\n",
    "        result=np.nanmean(view,axis=(2,3))\n",
    "\n",
    "    return result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Global Dataset store & Dummy set generation</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    datasets\n",
    "except:\n",
    "    datasets = {}\n",
    "\n",
    "name = 'Dummy'\n",
    "L = 1000\n",
    "_1,_2 = list(np.random.random((L,2))), list(np.random.random((L,2)))\n",
    "X1,X2 = [],[]\n",
    "Y1,Y2 = [],[]\n",
    "rad = 0.8\n",
    "for i in range(L):\n",
    "    a,b = _1[i][0],_1[i][1]\n",
    "    if a**2+b**2<rad**2:\n",
    "        Y1.append([1,0])\n",
    "        X1.append(_1[i])\n",
    "    elif a**2+b**2>=rad**2:\n",
    "        Y1.append([0,1])\n",
    "        X1.append(_1[i])\n",
    "    a,b = _2[i][0],_2[i][1]\n",
    "    if a**2+b**2<rad**2:\n",
    "        Y2.append([1,0])\n",
    "        X2.append(_2[i])\n",
    "    elif a**2+b**2>=rad**2:\n",
    "        Y2.append([0,1])\n",
    "        X2.append(_2[i])\n",
    "X1 = np.array(X1)\n",
    "X2 = np.array(X2)\n",
    "Y1 = np.array(Y1)\n",
    "Y2 = np.array(Y2)\n",
    "datasets[name] = (X1,Y1,['In','Out'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "m, n = 5,5\n",
    "X = np.array( list(it.product(np.arange(m),np.arange(n))) )\n",
    "Y = np.array( cattooht( np.ravel( ((np.array([list(np.arange(n))]*m).T+np.arange(m)).T)%2 ) )[0] )\n",
    "datasets['XOR'] = (X,Y,['E','O'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Loading MNIST & Cat-Dog datasets</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = 'data'\n",
    "res_path = os.getcwd()\n",
    "\n",
    "os.chdir(path)\n",
    "for fldr in os.listdir():\n",
    "    if not fldr.startswith('.'):\n",
    "        datasets[fldr] = ([],[])\n",
    "        os.chdir(fldr)\n",
    "        _ = sorted([x for x in os.listdir() if not x.startswith('.')])\n",
    "        name_index = {x:_.index(x) for x in _}\n",
    "        for category in _:\n",
    "            label = [0]*len(_)\n",
    "            label[name_index[category]] = 1\n",
    "            os.chdir(category)\n",
    "            for sample in os.listdir(): #[:2000]:\n",
    "                if not fldr.startswith('.'):\n",
    "                    img_mat = io.imread(sample, as_gray=True)\n",
    "                    if fldr=='Cat-Dog': img_mat = poolingOverlap(img_mat,(4,4))\n",
    "                    img_mat = np.ravel(img_mat)\n",
    "                    datasets[fldr][0].append(img_mat)\n",
    "                    datasets[fldr][1].append(label)\n",
    "            os.chdir('..')\n",
    "        datasets[fldr] = tuple(map(np.array,datasets[fldr]))+(_,)\n",
    "        os.chdir('..')\n",
    "\n",
    "os.chdir( res_path )\n",
    "for i in datasets:\n",
    "    datasets[i] = np.array(datasets[i][0],dtype='float64'), datasets[i][1], datasets[i][2]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Back-propagation Algorithm for Neural Network</h3>\n",
    "<img src=\"Images/BP.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Neural Network Class</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "class NN:\n",
    "    def __init__(self):\n",
    "        self.Num, self.fun = [], []\n",
    "        self.IP, self.OP, self.W, self.B, self.delta = {}, {}, {}, {}, {}\n",
    "        self.beta1, self.beta2, self.eps = 0.9, 0.999, 10**-8\n",
    "\n",
    "    def data_feed( self, M, L, targets):\n",
    "        self.raw, self.labels, self.target_names = M, L, targets\n",
    "\n",
    "    def data_validate( self, M=np.array([]), L=np.array([]) ):\n",
    "        self.vraw, self.vlabels = M, L\n",
    "\n",
    "    def add(self,N,f='relu'):\n",
    "        self.Num.append(N); self.fun.append(f)\n",
    "\n",
    "    def data_preprocess(self,mode='standard'):\n",
    "        sp = np.nan_to_num\n",
    "        try:\n",
    "            mode = self.preprocess_mode\n",
    "        except:\n",
    "            self.preprocess_mode = mode\n",
    "        if mode=='scale':\n",
    "            try:\n",
    "                self.mn, self.mx\n",
    "            except:\n",
    "                self.mn, self.mx = self.raw.min(axis=0), self.raw.max(axis=0)\n",
    "            mx = np.where(self.mx==self.mn,self.mx+1,self.mx)\n",
    "            self.data  = sp((self.raw - self.mn)/(mx-self.mn))\n",
    "            try: # If validation data is defined\n",
    "                self.vdata = sp((self.vraw - self.mn)/(self.mx-self.mn))\n",
    "            except:\n",
    "                self.vdata = self.data\n",
    "        elif mode=='standard':\n",
    "            try:\n",
    "                self.mean, self.std\n",
    "            except:\n",
    "                self.mean, self.std   = self.raw.mean(axis=0), self.raw.std(axis=0)\n",
    "            std = np.where(self.std==0,1,self.std)\n",
    "            self.data = sp((self.raw-self.mean)/std)\n",
    "            try: # If validation data is defined\n",
    "                self.vdata  =  sp((self.vraw-self.mean)/std)\n",
    "            except:\n",
    "                self.vdata = self.data\n",
    "        else:\n",
    "            raise Exception('Code should be unreachable')\n",
    "    \n",
    "    def initialize_layers(self,He=True,mode='gaussian'):\n",
    "        for i in range(len(self.Num)):\n",
    "            if i==0:\n",
    "                self.W[i],self.B[i], = initWB(self.data.shape[1],self.Num[i],self.fun[i],He,mode)\n",
    "            else:\n",
    "                self.W[i],self.B[i], = initWB(self.Num[i-1],self.Num[i],self.fun[i],He,mode)\n",
    "                \n",
    "    def forward_prop(self,predict=False):\n",
    "        self.IP[0] = self.fdata\n",
    "        for i in range(len(self.Num)):\n",
    "            wx_b = np.dot(self.IP[i],self.W[i])+self.B[i]\n",
    "            if not predict:\n",
    "                self.OP[i] = wx_b\n",
    "            _ = eval('ActV.{0}(wx_b)'.format(self.fun[i]))\n",
    "            self.IP[i+1] = _\n",
    "            if predict:\n",
    "                del self.IP[i]\n",
    "        return self.IP[len(self.Num)]\n",
    "\n",
    "    def back_prop(self,debug=False):\n",
    "        for i in range(len(self.Num)-1,-1,-1):\n",
    "            if debug: print('Layer',i)\n",
    "            if i==(len(self.Num)-1):\n",
    "                costD = eval('CostD.{0}(self.flabels,self.IP[len(self.Num)])'.format(self.cost))\n",
    "                actvD = eval('ActD.{0}(self.OP[i])'.format(self.fun[i]))\n",
    "                self.delta[i] = costD * actvD\n",
    "                if debug: print('>>',self.IP[i].shape,costD.shape,actvD.shape,self.delta[i].shape)\n",
    "            else:\n",
    "                costD = np.dot(self.W[i+1],self.delta[i+1].T).T # ((6,2),(100,2).T).T => (100,6)\n",
    "                actvD = eval('ActD.{0}(self.OP[i])'.format(self.fun[i])) #(100,6)\n",
    "                self.delta[i] = costD * actvD\n",
    "                if debug: print('>>',self.IP[i].shape,costD.shape,actvD.shape,self.delta[i].shape)\n",
    "            uW = np.dot( self.IP[i].T , self.delta[i] ) / self.IP[i].shape[0]\n",
    "            uB = np.mean( self.delta[i] ,axis=0, keepdims=True)\n",
    "            if debug: print( self.W[i].shape , self.B[i].shape)\n",
    "            if debug: print( uW.shape , uB.shape)\n",
    "            self.W[i] -= self.learning_rate*uW\n",
    "            self.B[i] -= self.learning_rate*uB\n",
    "            if debug: input()\n",
    "\n",
    "    def back_prop2(self,Iteration_Count=1,debug=False,amsgrad=False):\n",
    "        if Iteration_Count==1:\n",
    "            self.UW, self.UB, self.SW, self.SB =  deepcopy(self.W), deepcopy(self.B), deepcopy(self.W), deepcopy(self.B)\n",
    "            for i in self.UW:\n",
    "                self.UW[i], self.UB[i], self.SW[i], self.SB[i] = 0*self.UW[i], 0*self.UB[i], 0*self.SW[i], 0*self.SB[i]\n",
    "        for i in range(len(self.Num)-1,-1,-1):\n",
    "            if i==(len(self.Num)-1):\n",
    "                costD = eval('CostD.{0}(self.flabels,self.IP[len(self.Num)])'.format(self.cost))\n",
    "                actvD = eval('ActD.{0}(self.OP[i])'.format(self.fun[i]))\n",
    "                self.delta[i] = costD * actvD\n",
    "            else:\n",
    "                costD = np.dot(self.W[i+1],self.delta[i+1].T).T\n",
    "                actvD = eval('ActD.{0}(self.OP[i])'.format(self.fun[i]))\n",
    "                self.delta[i] = costD * actvD\n",
    "            uW = np.dot( self.IP[i].T , self.delta[i] ) / self.IP[i].shape[0]\n",
    "            uB = np.mean( self.delta[i] ,axis=0, keepdims=True)\n",
    "            # Eqn 1\n",
    "            self.UW[i] = self.beta1*self.UW[i] + (1-self.beta1)*uW\n",
    "            self.UB[i] = self.beta1*self.UB[i] + (1-self.beta1)*uB\n",
    "            # Eqn 2\n",
    "            self.SW[i] = self.beta2*self.SW[i] + (1-self.beta2)*uW**2\n",
    "            self.SB[i] = self.beta2*self.SB[i] + (1-self.beta2)*uB**2\n",
    "            # Eqn 3\n",
    "            UW = self.UW[i]/(1-self.beta1**Iteration_Count)\n",
    "            UB = self.UB[i]/(1-self.beta1**Iteration_Count)\n",
    "            # Eqn 4\n",
    "            SW = self.SW[i]/(1-self.beta2**Iteration_Count)\n",
    "            SB = self.SB[i]/(1-self.beta2**Iteration_Count)\n",
    "            # Eqn 5\n",
    "            self.W[i] -= self.learning_rate*UW/(SW**0.5+self.eps)\n",
    "            self.B[i] -= self.learning_rate*UB/(SB**0.5+self.eps)\n",
    "            if np.isnan(self.W[i]).any() or np.isnan(self.B[i]).any():\n",
    "                raise Exception('NAN value arises')\n",
    "\n",
    "    def back_prop3(self,Epoch_Count=1,debug=False):\n",
    "        for i in range(len(self.Num)-1,-1,-1):\n",
    "            if i==(len(self.Num)-1):\n",
    "                costD = eval('CostD.{0}(self.flabels,self.IP[len(self.Num)])'.format(self.cost))\n",
    "                actvD = eval('ActD.{0}(self.OP[i])'.format(self.fun[i]))\n",
    "                self.delta[i] = costD * actvD\n",
    "            else:\n",
    "                costD = np.dot(self.W[i+1],self.delta[i+1].T).T\n",
    "                actvD = eval('ActD.{0}(self.OP[i])'.format(self.fun[i]))\n",
    "                self.delta[i] = costD * actvD\n",
    "            uW = np.dot( self.IP[i].T , self.delta[i] ) / self.IP[i].shape[0]\n",
    "            uB = np.mean( self.delta[i] ,axis=0, keepdims=True)\n",
    "            # Eqn 1\n",
    "            _W1 = (1-self.beta1)*uW/(1-self.beta1**Epoch_Count)\n",
    "            _B1 = (1-self.beta1)*uB/(1-self.beta1**Epoch_Count)\n",
    "            # Eqn 2\n",
    "            _W2 = (1-self.beta2)*uW**2/(1-self.beta2**Epoch_Count)\n",
    "            _B2 = (1-self.beta2)*uB**2/(1-self.beta2**Epoch_Count)\n",
    "            # Eqn 3\n",
    "            self.W[i] -= self.learning_rate*_W1/(_W2**0.5+self.eps)\n",
    "            self.B[i] -= self.learning_rate*_B1/(_B2**0.5+self.eps)\n",
    "            if np.isnan(self.W[i]).any() or np.isnan(self.B[i]).any():\n",
    "                raise Exception('NAN value arises')\n",
    "\n",
    "    def feed_adam(beta1, beta2, eps):\n",
    "        self.beta1, self.beta2, self.eps = beta1, beta2, eps\n",
    "\n",
    "    def plot_feed(self,feed=True):\n",
    "        self.fdata,self.flabels = self.data, self.labels\n",
    "        y_pred = self.forward_prop(predict=True)\n",
    "        costV  = eval('CostV.{0}(self.flabels,y_pred)'.format(self.cost))\n",
    "        y_pred = one_hot(y_pred)\n",
    "        mvalue = eval('Metrices.{0}(self.flabels,y_pred)'.format(self.metric))\n",
    "        act2 = [ list(rw).index(1) for rw in self.flabels ]\n",
    "        pred2 = [ list(rw).index(1) for rw in y_pred ]\n",
    "        if feed:\n",
    "            self.costs.append( np.mean(costV) )\n",
    "            self.mvalues.append( mvalue )\n",
    "            self.f1m.append( f1_score(act2,pred2,average='micro') )\n",
    "            self.f1M.append( f1_score(act2,pred2,average='macro') )\n",
    "            \n",
    "            self.fdata,self.flabels = self.vdata, self.vlabels\n",
    "            y_pred = one_hot( self.forward_prop(predict=True) )\n",
    "            vmvalue = eval('Metrices.{0}(self.flabels,y_pred)'.format(self.metric))\n",
    "            self.vmvalues.append( vmvalue )\n",
    "            \n",
    "        return act2, pred2\n",
    "\n",
    "\n",
    "    def train(self,epochs=1000,batchsize=30,learning_rate=0.001,\\\n",
    "              optimizer='adam',cost='cross_entropy',metric='accuracy',es=(True,0,True),amsgrad=False):\n",
    "        \n",
    "        self.cost, self.metric, self.learning_rate = cost, metric, learning_rate\n",
    "        self.costs, self.mvalues, self.f1m, self.f1M, self.vmvalues = [], [], [], [], []\n",
    "        if es[0]: prev_entropy = [np.inf]\n",
    "        # Random value at starting NN\n",
    "        self.plot_feed()\n",
    "        f = open('continue_next_epoch','w')\n",
    "        f.close()\n",
    "\n",
    "        for T in range(epochs):\n",
    "            if 'continue_next_epoch' not in os.listdir(): break\n",
    "            init = datetime.now()\n",
    "            print('Epoch {0:{1}} ['.format(T+1,int(np.log10(epochs+1))+1),end='')\n",
    "            if es[0]: W,B = [deepcopy(self.W)],[deepcopy(self.B)] # Saving Weights for Early Stopping\n",
    "            mb_indx, splits = 0, int(np.ceil(self.data.shape[0]/batchsize))\n",
    "            self.index_set = Split(self.data, self.labels, splits ,'R')\n",
    "            for ln in range(len(self.index_set)):\n",
    "                train_indx, test_indx = self.index_set[ln]\n",
    "                self.fdata,self.flabels = self.data[test_indx],self.labels[test_indx]\n",
    "                self.forward_prop()\n",
    "                if optimizer=='gd':\n",
    "                    self.back_prop()\n",
    "                elif optimizer=='adam':\n",
    "                    self.back_prop2(T*len(self.index_set)+(ln+1))\n",
    "                else:\n",
    "                    self.back_prop3(T+1)\n",
    "                if(mb_indx>=(splits*0.04)):\n",
    "                    print('=',end='')\n",
    "                    mb_indx = 0\n",
    "                mb_indx+=1\n",
    "            # Early Stopping using Validation Set #CHECKPOINT\n",
    "            if es[0]:\n",
    "                if es[1]==-1:\n",
    "                    pass\n",
    "                else:\n",
    "                    delta = 0 # Exploring with compromising observed value\n",
    "                    self.fdata,self.flabels = self.vdata,self.vlabels\n",
    "                    y_pred = self.forward_prop(predict=True)\n",
    "                    costV  = eval('CostV.{0}(self.flabels,y_pred)'.format(self.cost))\n",
    "                    best_entropy, cur_entropy = min(prev_entropy), np.mean(costV)\n",
    "                    if ( cur_entropy - best_entropy) > delta :\n",
    "                        if len(prev_entropy)==(es[1]+1):\n",
    "                            if es[2]: # Restoring Best Weights\n",
    "                                bst_indx = len(prev_entropy)-prev_entropy[::-1].index(best_entropy) - 1\n",
    "                                self.W,self.B = W[bst_indx], B[bst_indx]\n",
    "                            print(']\\n',best_entropy,'==>',cur_entropy)\n",
    "                            break\n",
    "                        else:\n",
    "                            prev_entropy.append( cur_entropy )\n",
    "                            W.append(deepcopy(self.W)); B.append(deepcopy(self.B))\n",
    "                    else:\n",
    "                        W,B = [deepcopy(self.W)],[deepcopy(self.B)]\n",
    "                        prev_entropy = [ cur_entropy ]\n",
    "            # To plot results for entire datasets\n",
    "            self.plot_feed()\n",
    "            print('] Loss {0:.6e}, Accuracy {1:.2f}%, Accuracy-V {2:.2f}%, Time {3:}'.format(self.costs[-1],self.mvalues[-1]*100,self.vmvalues[-1]*100,datetime.now()-init))\n",
    "\n",
    "    def krs(self,epochs=1000,batchsize=30,learning_rate=0.001,\\\n",
    "              optimizer='adam',cost='cross_entropy',metric='accuracy',es=(True,0,True)):\n",
    "        model = Sequential()\n",
    "        addswish(model)\n",
    "        model.add(Dense(self.Num[0], activation=self.fun[0], input_dim=self.data.shape[1]))\n",
    "        for i in range(1,len(self.Num)-1):\n",
    "            model.add(Dense(self.Num[i], activation=self.fun[i]))\n",
    "        model.add(Dense(self.labels.shape[1], activation='softmax'))\n",
    "        cb = [EarlyStopping(monitor='val_loss', patience=es[1], restore_best_weights=es[2])] if (es[0] and es[1]!=-1) else []\n",
    "        model.compile(loss='categorical_crossentropy', optimizer=Adam(lr=learning_rate,amsgrad=False), metrics=[metric])\n",
    "        model.fit(self.data, self.labels, epochs=epochs, batch_size=batchsize,\\\n",
    "                  validation_data=(self.vdata, self.vlabels), callbacks=cb )\n",
    "        y_pred = model.predict(self.vdata)\n",
    "        y_pred = one_hot(y_pred)\n",
    "        self.kmodel = model\n",
    "        return classification_report(self.vlabels, y_pred, target_names=self.target_names, digits = 4 )\n",
    "\n",
    "    def report(self,model=None):\n",
    "        if model:\n",
    "            y_true1, y_pred1 = self.labels,  one_hot(model.predict(self.data))\n",
    "            y_true2, y_pred2 = self.vlabels, one_hot(model.predict(self.vdata))\n",
    "        else:\n",
    "            self.fdata, self.flabels = self.data, self.labels\n",
    "            y_true1, y_pred1 = self.labels, one_hot( self.forward_prop(predict=True) )\n",
    "            self.fdata, self.flabels = self.vdata, self.vlabels\n",
    "            y_true2, y_pred2 = self.vlabels, one_hot( self.forward_prop(predict=True) )\n",
    "            \n",
    "        r1 = classification_report(y_true1, y_pred1, target_names=self.target_names, digits = 4 )\n",
    "        r2 = classification_report(y_true2, y_pred2, target_names=self.target_names, digits = 4 )\n",
    "        return r1, r2\n",
    "\n",
    "    def plot(self,prms={},learning_plot=False):\n",
    "        mpl.rcParams['figure.dpi'] = 100\n",
    "        plt.close()\n",
    "        ax = plt.subplot(111)\n",
    "\n",
    "        ls = [ self.mvalues[1:], self.f1M[1:], self.f1m[1:], self.vmvalues[1:] ]\n",
    "        c1, c2 = min((min(l) for l in ls)), max((max(l) for l in ls))\n",
    "        _ = (np.array(self.costs[1:])-min(self.costs[1:])) / (max(self.costs[1:])-min(self.costs[1:]))\n",
    "        _ = list( (_*(c2-c1)+c1) )\n",
    "        ls = [_]+ls\n",
    "        for i in range(len(ls)):\n",
    "            ls[i] = np.array(ls[i])\n",
    "        s = np.exp(-5)\n",
    "        if learning_plot:\n",
    "            for i in range(len(ls)):\n",
    "                ls[i] = -np.log((c2+s)-np.array(ls[i]))\n",
    "            # Best Depiction of Learning process\n",
    "            indx   = list( np.linspace(-np.log((c2+s)-c1),-np.log((c2+s)-c2),10) )\n",
    "            yticks = np.round(np.linspace(c1,c2,10),3)\n",
    "\n",
    "        _1 = plt.plot(np.arange(1,len(ls[0])+1), ls[0],'-',label=self.cost)\n",
    "        _2 = plt.plot(np.arange(1,len(ls[1])+1), ls[1],'*',label='Accuracy-T')\n",
    "        _3 = plt.plot(np.arange(1,len(ls[2])+1), ls[2],'-.',label='F1-Macro')\n",
    "        _4 = plt.plot(np.arange(1,len(ls[3])+1), ls[3],':',label='F1-Micro')\n",
    "        _5 = plt.plot(np.arange(1,len(ls[4])+1), ls[4],'--',label='Accuracy-V')\n",
    "\n",
    "        if learning_plot:\n",
    "            plt.yticks(indx,yticks)\n",
    "\n",
    "        p1 = '{0} Accuracy {1:.2f}%'.format(self.name,(self.mvalues[-1]*0.9+self.vmvalues[-1]*0.1)*100)\n",
    "        prms = {x:prms[x] for x in prms if x in grid_params}\n",
    "        p2 = ', '.join(str(x) for x in tuple(prms[x] for x in grid_params) ) # Grid Search Hyperparameters\n",
    "        title = '\\n'.join((p1,p2))\n",
    "        plt.title(title)\n",
    "        plt.xlabel('Epochs')\n",
    "        plt.legend(loc=0)\n",
    "        plt.savefig(title+'.png',dpi=300,bbox_inches = 'tight')\n",
    "        if verbose:\n",
    "            plt.show()\n",
    "        plt.close()\n",
    "        \n",
    "    def missed(self,diff_validation=True):\n",
    "        try:\n",
    "            shutil.rmtree('missed')\n",
    "        except:\n",
    "            pass\n",
    "        finally:\n",
    "            os.mkdir('missed')\n",
    "            os.chdir('missed')\n",
    "        try:\n",
    "            ls = [(self.data,self.labels)]\n",
    "            if diff_validation:\n",
    "                ls.append( (self.vdata,self.vlabels) )\n",
    "            for data,labels in ls:\n",
    "                self.fdata, self.flabels = deepcopy(data), deepcopy(labels)\n",
    "                pred = one_hot( self.forward_prop(predict=True) )\n",
    "                act  = self.flabels\n",
    "                count = {}\n",
    "                for i in range(len(self.fdata)):\n",
    "                    if not (act[i]==pred[i]).all():\n",
    "                        lbl_a = self.target_names[ np.sum( act[i]*np.arange(act[i].shape[0])) ]\n",
    "                        lbl_p = self.target_names[ np.sum(pred[i]*np.arange(pred[i].shape[0])) ]\n",
    "                        if (lbl_a,lbl_p) in count:\n",
    "                            count[(lbl_a,lbl_p)]+=1\n",
    "                        else:\n",
    "                            count[(lbl_a,lbl_p)]=1\n",
    "                        mat = deepcopy( self.fdata[i] )\n",
    "                        try:\n",
    "                            mat = mat*(self.mx-self.mn)+self.mn\n",
    "                        except:\n",
    "                            mat = mat*self.std+self.mean\n",
    "                        mat = mat.reshape(round(mat.shape[0]**0.5),round(mat.shape[0]**0.5))\n",
    "                        mpl.image.imsave('{1},{2},{0}.png'.format(count[(lbl_a,lbl_p)],lbl_a,lbl_p),mat)\n",
    "        except:\n",
    "            pass\n",
    "        finally:\n",
    "            os.chdir('..')\n",
    "        \n",
    "    def save_model(self,model_store='models'):\n",
    "        if model_store not in os.listdir():\n",
    "            os.mkdir(model_store)\n",
    "        try:\n",
    "            try:\n",
    "                shutil.rmtree('{}/{}'.format(model_store,self.name))\n",
    "            except:\n",
    "                pass\n",
    "            finally:\n",
    "                os.mkdir('{}/{}'.format(model_store,self.name))\n",
    "                os.chdir('{}/{}'.format(model_store,self.name))\n",
    "            with open('config','w') as f:\n",
    "                print(repr(self.Num) ,file=f)\n",
    "                print(repr(self.fun) ,file=f)\n",
    "                print(self.preprocess_mode,end = '',file=f)                \n",
    "            dct = {}\n",
    "            with open('parameters','wb') as f:\n",
    "                if self.preprocess_mode == 'standard':\n",
    "                    dct['mean'], dct['std'] = self.mean, self.std\n",
    "                elif self.preprocess_mode == 'scale':\n",
    "                    dct['mn'], dct['mx'] = self.mn, self.mx\n",
    "                else:\n",
    "                    raise Exception('Code should be unreachable')\n",
    "                for i in self.W:\n",
    "                    dct['W{}'.format(i)] = self.W[i]\n",
    "                    dct['B{}'.format(i)] = self.B[i]\n",
    "                np.savez(f,**dct)\n",
    "        except Exception as exc:\n",
    "            pass\n",
    "        finally:\n",
    "            os.chdir('../..')\n",
    "    \n",
    "    def load_model(self,model_store = 'models'):\n",
    "        if model_store not in os.listdir():\n",
    "            raise Exception(\"{} directory does not Exist\".format(model_store))\n",
    "        try:\n",
    "            os.chdir('{}/{}'.format(model_store,self.name))\n",
    "            with open('config') as f:\n",
    "                self.Num = eval(f.readline().strip())\n",
    "                self.fun = eval(f.readline().strip())\n",
    "                self.preprocess_mode = f.readline().strip()\n",
    "            with open('parameters','rb') as f:\n",
    "                npzfile = np.load(f)\n",
    "                if self.preprocess_mode == 'standard':\n",
    "                    self.mean, self.std = npzfile['mean'], npzfile['std']\n",
    "                elif self.preprocess_mode == 'scale':\n",
    "                    self.mn, self.mx = npzfile['mn'], npzfile['mx']\n",
    "                else:\n",
    "                    raise Exception('Code should be unreachable')\n",
    "                for i in range(len(self.Num)):\n",
    "                    self.W[i] = npzfile['W{}'.format(i)]\n",
    "                    self.B[i] = npzfile['B{}'.format(i)]\n",
    "        except Exception as exc:\n",
    "            pass\n",
    "        finally:\n",
    "            os.chdir('../..')\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Dataset Evaluation Wrapper</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Early Stopping Parameters\n",
    "# (Enable, ValidationPartition, Patience, Restore)\n",
    "\n",
    "def evaldata(name,NumFun,prprc='standard',He=True,initmode='gaussian',\\\n",
    "             epochs=1000,batchsize=30,lr=0.001,opt='adam',es=(True,False,0,True),krs=True):\n",
    "    params = locals()\n",
    "    if 'grid_params' not in globals():\n",
    "        global grid_params\n",
    "        grid_params = []\n",
    "\n",
    "    print('Dataset under processing: ',name)\n",
    "    X,Y,targets = datasets[name]\n",
    "    net = NN()\n",
    "    net.name = name\n",
    "    if es[1]:\n",
    "        index_set = SSplit(X,Y,10)\n",
    "        np.random.shuffle(index_set)\n",
    "        train_index, test_index = index_set[0]\n",
    "        X1, Y1, X2, Y2 = X[train_index], Y[train_index], X[test_index], Y[test_index]\n",
    "    else:\n",
    "        X1, Y1, X2, Y2 = X, Y, X, Y\n",
    "    net.data_feed(X1,Y1,targets) # Feeding Raw data\n",
    "    net.data_validate(X2,Y2)     # Used for Early Stopping\n",
    "    net.data_preprocess(prprc)\n",
    "    #Adding Hidden Layers\n",
    "    for n,f in NumFun:\n",
    "        net.add(n,f)        \n",
    "    # Output Layer & Cost function\n",
    "    net.add(Y.shape[1],'softmax')\n",
    "    net.initialize_layers(He,initmode)\n",
    "    # Calling Training module, with optmizer & regularization parameters\n",
    "    print('\\n\\t\\t','#'*16,'NumPy Implementation','#'*16,'\\n')\n",
    "    net.train( epochs, batchsize, lr, opt, 'cross_entropy', 'accuracy', es[0:1]+es[2:] )\n",
    "    r1, r2 = net.report()\n",
    "    print('\\n\\t\\t\\t','-'*8,'Classification Report on Training data','-'*8,'\\n',r1)\n",
    "    if es[1]: print('\\n\\t\\t','-'*8,'Classification Report on Validation data','-'*8,'\\n',r2)\n",
    "    if krs:\n",
    "        print('\\n\\t\\t','#'*16,'Keras Implementation','#'*16,'\\n')\n",
    "        net.krs( epochs, batchsize, lr, opt, 'cross_entropy', 'accuracy', es[0:1]+es[2:] )\n",
    "        r1, r2 = net.report(net.kmodel)\n",
    "        print('\\n\\t\\t\\t','-'*8,'Classification Report on Training data','-'*8,'\\n',r1)\n",
    "        if es[1]: print('\\n\\t\\t','-'*8,'Classification Report on Validation data','-'*8,'\\n',r2)\n",
    "\n",
    "    net.plot(params) \n",
    "    if krs:\n",
    "        net.missed(es[1])\n",
    "    net.save_model()\n",
    "    net.load_model()\n",
    "    return net.mvalues, net.costs, net.f1M, net.f1m"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Grid Search for Hyper-parameter tuning</h3>\n",
    "<pre>\n",
    "<b>Sample Worst Case Sample given below, 9k+ executions</b><br>\n",
    "    ########################################################\n",
    "    ################ DON'T TRY THIS AT HOME ################\n",
    "    ########################################################\n",
    "\n",
    "    dct = {\n",
    "        'A datasets'       : ['Dummy'],\n",
    "        'B units'          : list(zip((392,784,1568),(64,128,128))),\n",
    "        'C functions'      : it.product(('sigmoid','tanh','relu','swish'),('sigmoid','tanh','relu','swish')),\n",
    "        'D preproc'        : ['scale','standard'],\n",
    "        'E He'             : [True,False],\n",
    "        'F initmodes'      : ['uniform','gaussian'],\n",
    "        'G epochs'         : [10,20,40],\n",
    "        'H batchsize'      : [128,256,512,1024],\n",
    "        'I learning_rate'  : [0.001,0.0003,0.0001],\n",
    "        'J optimizer'      : ['adam'],\n",
    "        'K early_stopping' : [(True,True,5,True)],\n",
    "        'L keras'          : [True],\n",
    "    }\n",
    "    res = grid_search(dct)\n",
    "    grid_plot(res)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def grid_search(dct):\n",
    "    grid_values = { 'Accuracy':{}, 'Cost':{}, 'F1-Macro':{}, 'F1-Micro':{} }\n",
    "    for prms in it.product(*(dct[x] for x in dct)):\n",
    "        name = prms[0]\n",
    "        prms = list(prms)\n",
    "        prms[1:3] = [tuple(zip(prms[1],prms[2]))]\n",
    "        prms = tuple(prms)\n",
    "        print('STARTED  ',prms)\n",
    "        _ = eval( 'evaldata{0}'.format(tuple(prms)))\n",
    "        print('COMPLETED',prms,end='\\n'*3)\n",
    "        for i in range(len(_)):\n",
    "            ls = sorted( grid_values.keys() )\n",
    "            grid_values[ls[i]][prms] = _[i][-1]\n",
    "    return grid_values\n",
    "\n",
    "def grid_plot(res,dct):\n",
    "    'Under assumption that only one quantity will be varied at a time'\n",
    "    tmp_grid_params = ['DataSet','Config','Preprocess','He','InitMode','Epochs','Batch_Size','Learning_Rate']\n",
    "    \n",
    "    def plot(metric,inner_dct,color):\n",
    "        param_vals, y_values = {i:set() for i in range(len(tmp_grid_params))}, []\n",
    "        for params in sorted(inner_dct):\n",
    "            y_values.append( inner_dct[params] )\n",
    "            for indx in range(len(tmp_grid_params)):\n",
    "                param_vals[ indx ].add(params[indx])\n",
    "\n",
    "        for indx in range(len(tmp_grid_params)):\n",
    "            if len(param_vals[indx])>1:\n",
    "                break\n",
    "        else: # No graph can be shown with no changing values\n",
    "            return\n",
    "\n",
    "        if tmp_grid_params[indx]=='Config':\n",
    "            sample = list(inner_dct.keys())[0][indx]\n",
    "            inner_indx_dct = {(i,j):set() for i in range(len(sample)) for j in range(2)}\n",
    "            for params in sorted(inner_dct):\n",
    "                for i in range(len(sample)):\n",
    "                    for j in range(2):\n",
    "                        inner_indx_dct[(i,j)].add(params[indx][i][j])\n",
    "            for inner_indx in inner_indx_dct:\n",
    "                if len(inner_indx_dct[inner_indx])>1:\n",
    "                    break\n",
    "            else:\n",
    "                return\n",
    "            i,j = inner_indx\n",
    "            par_name = 'Layer {0}'.format(i+1)+' '+('Activation' if j else 'Units')\n",
    "            x_values = sorted(inner_indx_dct[inner_indx])\n",
    "        else:\n",
    "            par_name = tmp_grid_params[indx]\n",
    "            x_values = [params[indx] for params in sorted(inner_dct)]\n",
    "                \n",
    "        ind = np.arange(len(y_values))\n",
    "        plt.xlabel(par_name)\n",
    "        plt.ylabel(metric)\n",
    "        plt.xticks( ind,x_values)\n",
    "        try:\n",
    "            styl = ('*' if set(map(int,x_values))=={0,1} else '-')\n",
    "        except:\n",
    "            styl = '*'\n",
    "        plt.plot(   ind,y_values,styl,color=color)\n",
    "        title = ', '.join((metric,par_name))\n",
    "        plt.title(title)\n",
    "        plt.savefig(title+'.png',dpi=300,bbox_inches = 'tight')\n",
    "        if verbose:\n",
    "            plt.show()\n",
    "        plt.close()\n",
    "\n",
    "    for metric,color in zip(res,('g','r','b','y')):\n",
    "        plt.close()\n",
    "        plot(metric,res[metric],color)\n",
    "        \n",
    "def multi_grid_search(dct,plot=False):\n",
    "    for i in dct:\n",
    "        if i=='next':\n",
    "            for dct2 in dct[i]:\n",
    "                multi_grid_search({dct2:dct[i][dct2]},plot)\n",
    "        else:\n",
    "            _ = os.getcwd()\n",
    "            try:\n",
    "                shutil.rmtree(i)\n",
    "            except:\n",
    "                pass\n",
    "            finally:\n",
    "                os.mkdir(i)\n",
    "                os.chdir(i)\n",
    "            print('\\n'+'#'*16+' Grid Search Started in {0} '.format(i)+'#'*16)\n",
    "            res = grid_search(dct[i])\n",
    "            try:\n",
    "                if plot: grid_plot(res,dct)\n",
    "            except Exception as exc:\n",
    "                print(exc)\n",
    "                pass\n",
    "            finally:\n",
    "                os.chdir(_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<pre>\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>Part 0 Pre-execution checks & Sample executions</h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sys.stdout = sys.__stdout__ = open('stdoutbuffer','a',buffering=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# os.chdir('..')\n",
    "os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "grid_params = ['NumFun','prprc','He', 'initmode', 'batchsize','lr']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "name,config = 'Dummy',[(4,'swish'),(3,'relu')]\n",
    "_ = evaldata(name,config,'standard',True,'gaussian',100,30,0.01,'adam',(True,True,-1,True),False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "name,config = 'XOR',[(10,'sigmoid'),(10,'sigmoid'),]\n",
    "_ = evaldata(name,config,'standard',True,'uniform',1000,1,0.001,'gd',(True,False,-1,True),False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "name,config = 'MNIST',[(1568,'swish'),(256,'swish'),]\n",
    "_1 = evaldata(name,config,'standard',True,'gaussian',10,2000,0.001,'myopt',(True,True,-1,True),False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "name,config = 'Cat-Dog',[(2048,'relu'),(256,'relu'),(64,'tanh')]\n",
    "_ = evaldata(name,config,'standard',True,'gaussian',100,200,0.0001,'myopt',(True,False,-1,True),False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<pre>\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "<h1>Part 1 \"MNIST\" Evaluation and Experiments</h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "grid_params = ['NumFun','prprc','He', 'initmode', 'batchsize','lr']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "<h3>Task 1 - Varying Number of Layers</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    shutil.rmtree('Task1')\n",
    "except:\n",
    "    pass\n",
    "finally:\n",
    "    os.mkdir('Task1')\n",
    "    os.chdir('Task1')\n",
    "\n",
    "grid_params = ['NumFun','prprc','He', 'initmode', 'batchsize','lr']\n",
    "name,config = 'MNIST',[(1568,'relu'),]\n",
    "_1 = evaldata(name,config,'standard',True,'gaussian',20,1000,0.001,'myopt',(True,True,5,True),False)\n",
    "name,config = 'MNIST',[(1568,'relu'),(256,'tanh')]\n",
    "_2 = evaldata(name,config,'standard',True,'gaussian',20,1000,0.001,'myopt',(True,True,5,True),False)\n",
    "name,config = 'MNIST',[(1568,'relu'),(256,'tanh'),(64,'tanh')]\n",
    "_3 = evaldata(name,config,'standard',True,'gaussian',20,1000,0.001,'myopt',(True,True,5,True),False)\n",
    "\n",
    "ls = ['Accuracy','F1-Macro','F1-Micro']\n",
    "color = ['green','blue','red']\n",
    "\n",
    "\n",
    "_ = [_1,_2,_3]\n",
    "for i in range(3):\n",
    "    _[i] = list(_[i])\n",
    "    _[i][:2] = _[i][:2][::-1]\n",
    "\n",
    "for i in range(3): #Metric\n",
    "    plt.close()\n",
    "    plt.title(ls[i])\n",
    "    plt.plot(list(range(1,3+1)),[_[j][i+1][-1] for j in range(3)],color=color[i])\n",
    "    plt.savefig(ls[i]+'1')\n",
    "    plt.close()\n",
    "    \n",
    "os.chdir('..')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part1/Task1/Accuracy1.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part1/Task1/F1Macro1.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part1/Task1/F1Micro1.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "<h3>Task 2 - Trying Various number of neurons in each layer</h3>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "<h5>SubTask 1 - Changing Number of Units in 1<sup>st</sup> Hidden Layer of Architecture</h5>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dct1 =  {\n",
    "    '1 Layer' :\n",
    "    {\n",
    "        'A datasets'       : ['MNIST'],\n",
    "        'B units'          : it.product((49,98,196,392,784,1176,1568,)),\n",
    "        'C functions'      : it.product(('relu',)),\n",
    "        'D preproc'        : ['standard'],\n",
    "        'E He'             : [True],\n",
    "        'F initmodes'      : ['gaussian'],\n",
    "        'G epochs'         : [20],\n",
    "        'H batchsize'      : [1000],\n",
    "        'I learning_rate'  : [0.001],\n",
    "        'J optimizer'      : ['myopt'],\n",
    "        'K early_stopping' : [(True,True,5,True)],\n",
    "        'L keras'          : [False],\n",
    "    }\n",
    "        }\n",
    "\n",
    "multi_grid_search(dct1,True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part1/Task2/1Layer/CostLayer1Units.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part1/Task2/1Layer/AccuracyLayer1Units.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part1/Task2/1Layer/F1MacroLayer1Units.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part1/Task2/1Layer/F1MicroLayer1Units.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "<h5>SubTask 2 - Changing Number of Units in 2<sup>nd</sup> Hidden Layer of Architecture</h5>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dct2 = {\n",
    "    '2 Layer' :\n",
    "    {\n",
    "        'A datasets'       : ['MNIST'],\n",
    "        'B units'          : it.product((1568,),(16,32,64,128,256)),\n",
    "        'C functions'      : it.product(('relu',),('tanh',)),\n",
    "        'D preproc'        : ['standard'],\n",
    "        'E He'             : [True],\n",
    "        'F initmodes'      : ['gaussian'],\n",
    "        'G epochs'         : [20],\n",
    "        'H batchsize'      : [1000],\n",
    "        'I learning_rate'  : [0.001],\n",
    "        'J optimizer'      : ['myopt'],\n",
    "        'K early_stopping' : [(True,True,5,True)],\n",
    "        'L keras'          : [False],\n",
    "    }\n",
    "        }\n",
    "\n",
    "multi_grid_search(dct2,True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part1/Task2/2Layer/CostLayer2Units.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part1/Task2/2Layer/AccuracyLayer2Units.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part1/Task2/2Layer/F1MacroLayer2Units.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part1/Task2/2Layer/F1MicroLayer2Units.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "<h5>SubTask 3 - Changing Number of Units in 3<sup>rd</sup> Hidden Layer of Architecture</h5>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dct3 = {\n",
    "    '3 Layer' :\n",
    "    {\n",
    "        'A datasets'       : ['MNIST'],\n",
    "        'B units'          : it.product((1568,),(256,),(16,32,64)),\n",
    "        'C functions'      : it.product(('relu',),('tanh',),('tanh',)),\n",
    "        'D preproc'        : ['standard'],\n",
    "        'E He'             : [True],\n",
    "        'F initmodes'      : ['gaussian'],\n",
    "        'G epochs'         : [20],\n",
    "        'H batchsize'      : [1000],\n",
    "        'I learning_rate'  : [0.001],\n",
    "        'J optimizer'      : ['myopt'],\n",
    "        'K early_stopping' : [(True,True,5,True)],\n",
    "        'L keras'          : [False],\n",
    "    }\n",
    "        }\n",
    "\n",
    "multi_grid_search(dct3,True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part1/Task2/3Layer/CostLayer3Units.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part1/Task2/3Layer/AccuracyLayer3Units.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part1/Task2/3Layer/F1MacroLayer3Units.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part1/Task2/3Layer/F1MicroLayer3Units.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "<h3>Task 3 - Trying Activation Functions on each layer</h3>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "<h5>SubTask 1 - Changing Activation Functions in 1<sup>st</sup> Hidden Layer of Architecture</h5>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dct1 =  {\n",
    "    '1 Layer FUNCTIONS' :\n",
    "    {\n",
    "        'A datasets'       : ['MNIST'],\n",
    "        'B units'          : it.product((1568,)),\n",
    "        'C functions'      : it.product(('sigmoid','relu','tanh','swish')),\n",
    "        'D preproc'        : ['standard'],\n",
    "        'E He'             : [True],\n",
    "        'F initmodes'      : ['gaussian'],\n",
    "        'G epochs'         : [20],\n",
    "        'H batchsize'      : [1000],\n",
    "        'I learning_rate'  : [0.001],\n",
    "        'J optimizer'      : ['myopt'],\n",
    "        'K early_stopping' : [(True,True,5,True)],\n",
    "        'L keras'          : [False],\n",
    "    }\n",
    "        }\n",
    "\n",
    "multi_grid_search(dct1,True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part1/Task3/1LayerF/CostLayer1Activation.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part1/Task3/1LayerF/AccuracyLayer1Activation.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part1/Task3/1LayerF/F1MacroLayer1Activation.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part1/Task3/1LayerF/F1MicroLayer1Activation.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "<h5>SubTask 2 - Changing Activation Functions in 2<sup>nd</sup> Hidden Layer of Architecture</h5>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dct2 = {\n",
    "    '2 Layer FUNCTIONS' :\n",
    "    {\n",
    "        'A datasets'       : ['MNIST'],\n",
    "        'B units'          : it.product((1568,),(256,)),\n",
    "        'C functions'      : it.product(('relu',),('sigmoid','relu','tanh','swish')),\n",
    "        'D preproc'        : ['standard'],\n",
    "        'E He'             : [True],\n",
    "        'F initmodes'      : ['gaussian'],\n",
    "        'G epochs'         : [20],\n",
    "        'H batchsize'      : [1000],\n",
    "        'I learning_rate'  : [0.001],\n",
    "        'J optimizer'      : ['myopt'],\n",
    "        'K early_stopping' : [(True,True,5,True)],\n",
    "        'L keras'          : [False],\n",
    "    }\n",
    "        }\n",
    "\n",
    "multi_grid_search(dct2,True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part1/Task3/2LayerF/CostLayer2Activation.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part1/Task3/2LayerF/AccuracyLayer2Activation.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part1/Task3/2LayerF/F1MacroLayer2Activation.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part1/Task3/2LayerF/F1MicroLayer2Activation.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "<h5>SubTask 3 - Changing Activation Functions in 3<sup>rd</sup> Hidden Layer of Architecture</h5>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dct3 = {\n",
    "    '3 Layer FUNCTIONS' :\n",
    "    {\n",
    "        'A datasets'       : ['MNIST'],\n",
    "        'B units'          : it.product((1568,),(256,),(64,)),\n",
    "        'C functions'      : it.product(('relu',),('tanh',),('relu','tanh','swish'),),\n",
    "        'D preproc'        : ['standard'],\n",
    "        'E He'             : [True],\n",
    "        'F initmodes'      : ['gaussian'],\n",
    "        'G epochs'         : [20],\n",
    "        'H batchsize'      : [1000],\n",
    "        'I learning_rate'  : [0.001],\n",
    "        'J optimizer'      : ['myopt'],\n",
    "        'K early_stopping' : [(True,True,5,True)],\n",
    "        'L keras'          : [False],\n",
    "    }\n",
    "        }\n",
    "\n",
    "multi_grid_search(dct3,True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part1/Task3/3LayerF/CostLayer3Activation.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part1/Task3/3LayerF/AccuracyLayer3Activation.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part1/Task3/3LayerF/F1MacroLayer3Activation.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part1/Task3/3LayerF/F1MicroLayer3Activation.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Task 4 Initialization & Preprocessing Techniques</h3>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "<h5>SubTask 1 Impact of Xavier-He weight Initiliazation</h5>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dct2 =  {\n",
    "    'Xavier-He':\n",
    "    {\n",
    "        'A datasets'       : ['MNIST'],\n",
    "        'B units'          : it.product((1568,),(256,)),\n",
    "        'C functions'      : it.product(('relu',),('tanh',)),\n",
    "        'D preproc'        : ['standard'],\n",
    "        'E He'             : [False,True],\n",
    "        'F initmodes'      : ['gaussian'],\n",
    "        'G epochs'         : [20],\n",
    "        'H batchsize'      : [1000],\n",
    "        'I learning_rate'  : [0.001],\n",
    "        'J optimizer'      : ['myopt'],\n",
    "        'K early_stopping' : [(True,True,5,True)],\n",
    "        'L keras'          : [False],\n",
    "    }\n",
    "        }\n",
    "\n",
    "multi_grid_search(dct2,True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h5>Observed learning curve for Initialization technique as \"Default\" vs \"Xavier-He\"</h5>\n",
    "<br><img src=\"output_plots/Part1/Task4/XH/NHE.png\" height=\"300\" width=\"450\" align=\"left\">\n",
    "<img src=\"output_plots/Part1/Task4/XH/YHE.png\" height=\"300\" width=\"450\" align=\"right\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "<h5>SubTask 2 Finding Suitable Preprocesing & Initialization Distirbution</h5>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dct1 =  {\n",
    "    'Init Gaussian':\n",
    "    {\n",
    "    'A datasets'       : ['MNIST'],\n",
    "    'B units'          : it.product((1568,),(256,)),\n",
    "    'C functions'      : it.product(('relu',),('tanh',)),\n",
    "    'D preproc'        : ['standard','scale'],\n",
    "    'E He'             : [True],\n",
    "    'F initmodes'      : ['gaussian'],\n",
    "    'G epochs'         : [20],\n",
    "    'H batchsize'      : [1000],\n",
    "    'I learning_rate'  : [0.001],\n",
    "    'J optimizer'      : ['myopt'],\n",
    "    'K early_stopping' : [(True,True,5,True)],\n",
    "    'L keras'          : [False],\n",
    "    },\n",
    "\n",
    "    'Init Uniform':\n",
    "    {\n",
    "    'A datasets'       : ['MNIST'],\n",
    "    'B units'          : it.product((1568,),(256,)),\n",
    "    'C functions'      : it.product(('relu',),('tanh',)),\n",
    "    'D preproc'        : ['standard','scale'],\n",
    "    'E He'             : [True],\n",
    "    'F initmodes'      : ['uniform'],\n",
    "    'G epochs'         : [20],\n",
    "    'H batchsize'      : [1000],\n",
    "    'I learning_rate'  : [0.001],\n",
    "    'J optimizer'      : ['myopt'],\n",
    "    'K early_stopping' : [(True,True,5,True)],\n",
    "    'L keras'          : [False],\n",
    "    }\n",
    "        }\n",
    "\n",
    "multi_grid_search(dct1,True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "<h5>Observed learning curve for Preprocessing from \"Scaling\" vs \"Standardization\"</h5>\n",
    "<br><img src=\"output_plots/Part1/Task4/IG/GraphG2.png\" height=\"300\" width=\"450\" align=\"left\">\n",
    "<img src=\"output_plots/Part1/Task4/IG/GraphG.png\" height=\"250\" width=\"450\" align=\"right\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "<b>Note: </b>Standard preprocessing, being insensitive to out-liers performs better than Min-Max Scaling<br>\n",
    "Hence, most our experiments shown use standardization, and might follow same for future ones"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "<h5>Observed learning curve for Initialization from \"Uniform\" vs \"Gaussian\"</h5>\n",
    "<br><img src=\"output_plots/Part1/Task4/IU/GraphU.png\" height=\"300\" width=\"450\" align=\"left\">\n",
    "<img src=\"output_plots/Part1/Task4/IG/GraphG.png\" height=\"250\" width=\"450\" align=\"right\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>General obervation:</b>\n",
    "Initialization when done from Gaussian distribution inserts minimum information in a system<br>\n",
    "Hence our most experiments will use it, you are free to experiment with other options"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Task 5 Comparing Classification Reports of Numpy & Keras implementations</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dct1 =  {\n",
    "    'Keras':\n",
    "    {\n",
    "        'A datasets'       : ['MNIST'],\n",
    "        'B units'          : it.product((1568,),(256,)),\n",
    "        'C functions'      : it.product(('relu',),('tanh',)),\n",
    "        'D preproc'        : ['standard'],\n",
    "        'E He'             : [True],\n",
    "        'F initmodes'      : ['gaussian'],\n",
    "        'G epochs'         : [20],\n",
    "        'H batchsize'      : [1000],\n",
    "        'I learning_rate'  : [0.001],\n",
    "        'J optimizer'      : ['myopt'],\n",
    "        'K early_stopping' : [(True,True,5,True)],\n",
    "        'L keras'          : [True],\n",
    "    },\n",
    "        }\n",
    "\n",
    "multi_grid_search(dct1,True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<pre>\n",
    "\n",
    "\t\t ################ NumPy Implementation ################ \n",
    "\n",
    "        -------- Classification Report on Training data -------- \n",
    "               precision    recall  f1-score   support\n",
    "\n",
    "           0     0.9984    0.9997    0.9991      3719\n",
    "           1     1.0000    0.9983    0.9992      4212\n",
    "           2     0.9989    0.9995    0.9992      3746\n",
    "           3     0.9992    0.9977    0.9985      3925\n",
    "           4     0.9986    0.9989    0.9988      3644\n",
    "           5     0.9985    0.9985    0.9985      3438\n",
    "           6     0.9992    0.9987    0.9989      3715\n",
    "           7     0.9977    0.9997    0.9987      3970\n",
    "           8     0.9986    0.9997    0.9992      3665\n",
    "           9     0.9981    0.9968    0.9975      3766\n",
    "\n",
    "   micro avg     0.9988    0.9988    0.9988     37800\n",
    "   macro avg     0.9987    0.9988    0.9988     37800\n",
    "weighted avg     0.9988    0.9988    0.9988     37800\n",
    " samples avg     0.9988    0.9988    0.9988     37800\n",
    "\n",
    "\n",
    "        -------- Classification Report on Validation data -------- \n",
    "               precision    recall  f1-score   support\n",
    "\n",
    "           0     0.9689    0.9806    0.9747       413\n",
    "           1     0.9871    0.9703    0.9786       472\n",
    "           2     0.9630    0.9652    0.9641       431\n",
    "           3     0.9240    0.9413    0.9326       426\n",
    "           4     0.9620    0.9463    0.9541       428\n",
    "           5     0.9660    0.9552    0.9606       357\n",
    "           6     0.9833    0.9787    0.9810       422\n",
    "           7     0.9578    0.9490    0.9534       431\n",
    "           8     0.9358    0.9523    0.9440       398\n",
    "           9     0.9343    0.9431    0.9387       422\n",
    "\n",
    "   micro avg     0.9583    0.9583    0.9583      4200\n",
    "   macro avg     0.9582    0.9582    0.9582      4200\n",
    "weighted avg     0.9585    0.9583    0.9584      4200\n",
    "\n",
    "\n",
    "\n",
    "\t\t ################ Keras Implementation ################ \n",
    "         \n",
    "         -------- Classification Report on Training data -------- \n",
    "               precision    recall  f1-score   support\n",
    "\n",
    "           0     0.9995    1.0000    0.9997      3719\n",
    "           1     0.9995    0.9991    0.9993      4212\n",
    "           2     0.9997    1.0000    0.9999      3746\n",
    "           3     0.9997    0.9987    0.9992      3925\n",
    "           4     0.9997    0.9997    0.9997      3644\n",
    "           5     0.9994    0.9997    0.9996      3438\n",
    "           6     0.9992    0.9997    0.9995      3715\n",
    "           7     0.9982    0.9995    0.9989      3970\n",
    "           8     0.9995    0.9992    0.9993      3665\n",
    "           9     0.9992    0.9981    0.9987      3766\n",
    "\n",
    "   micro avg     0.9994    0.9994    0.9994     37800\n",
    "   macro avg     0.9994    0.9994    0.9994     37800\n",
    "weighted avg     0.9994    0.9994    0.9994     37800\n",
    " samples avg     0.9994    0.9994    0.9994     37800\n",
    "\n",
    "\n",
    "\t\t -------- Classification Report on Validation data -------- \n",
    "               precision    recall  f1-score   support\n",
    "\n",
    "           0     0.9806    0.9782    0.9794       413\n",
    "           1     0.9850    0.9746    0.9798       472\n",
    "           2     0.9501    0.9722    0.9610       431\n",
    "           3     0.9307    0.9460    0.9383       426\n",
    "           4     0.9553    0.9486    0.9519       428\n",
    "           5     0.9624    0.9328    0.9474       357\n",
    "           6     0.9833    0.9739    0.9786       422\n",
    "           7     0.9471    0.9559    0.9515       431\n",
    "           8     0.9340    0.9598    0.9467       398\n",
    "           9     0.9444    0.9265    0.9354       422\n",
    "\n",
    "   micro avg     0.9574    0.9574    0.9574      4200\n",
    "   macro avg     0.9573    0.9569    0.9570      4200\n",
    "weighted avg     0.9576    0.9574    0.9574      4200\n",
    " samples avg     0.9574    0.9574    0.9574      4200"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<pre>\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "<h1>Part 2 \"Cat-Dog\" Evaluation and Experiments</h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "grid_params = ['NumFun','prprc','He', 'initmode', 'batchsize','lr']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "<h3>Task 1 - Varying Number of Layers</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    shutil.rmtree('Task1')\n",
    "except:\n",
    "    pass\n",
    "finally:\n",
    "    os.mkdir('Task1')\n",
    "    os.chdir('Task1')\n",
    "\n",
    "grid_params = ['NumFun','prprc','He', 'initmode', 'batchsize','lr']\n",
    "name,config = 'Cat-Dog',[(2048,'relu'),]\n",
    "_1 = evaldata(name,config,'standard',True,'gaussian',100,200,0.0001,'myopt',(True,True,-1,True),False)\n",
    "name,config = 'Cat-Dog',[(2048,'relu'),(256,'relu')]\n",
    "_2 = evaldata(name,config,'standard',True,'gaussian',100,200,0.0001,'myopt',(True,True,-1,True),False)\n",
    "name,config = 'Cat-Dog',[(2048,'relu'),(256,'relu'),(64,'tanh')]\n",
    "_3 = evaldata(name,config,'standard',True,'gaussian',100,200,0.0001,'myopt',(True,True,-1,True),False)\n",
    "\n",
    "ls = ['Accuracy','F1-Macro','F1-Micro']\n",
    "color = ['green','blue','red']\n",
    "\n",
    "\n",
    "_ = [_1,_2,_3]\n",
    "for i in range(3):\n",
    "    _[i] = list(_[i])\n",
    "    _[i][:2] = _[i][:2][::-1]\n",
    "\n",
    "for i in range(3): #Metric\n",
    "    plt.close()\n",
    "    plt.title(ls[i])\n",
    "    plt.plot(list(range(1,3+1)),[_[j][i+1][-1] for j in range(3)],color=color[i])\n",
    "    plt.savefig(ls[i]+'1')\n",
    "    plt.close()\n",
    "    \n",
    "os.chdir('..')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part2/Task1/Accuracy1.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part2/Task1/F1Macro1.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part2/Task1/F1Micro1.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "<h3>Task 2 - Trying Various number of neurons in each layer</h3>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "<h5>SubTask 1 - Changing Number of Units in 1<sup>st</sup> Hidden Layer of Architecture</h5>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dct1 =  {\n",
    "    '1 Layer' :\n",
    "    {\n",
    "        'A datasets'       : ['Cat-Dog'],\n",
    "        'B units'          : it.product((512,1024,2048)),\n",
    "        'C functions'      : it.product(('relu',)),\n",
    "        'D preproc'        : ['standard'],\n",
    "        'E He'             : [True],\n",
    "        'F initmodes'      : ['gaussian'],\n",
    "        'G epochs'         : [100],\n",
    "        'H batchsize'      : [200],\n",
    "        'I learning_rate'  : [0.0001],\n",
    "        'J optimizer'      : ['myopt'],\n",
    "        'K early_stopping' : [(True,True,-1,True)],\n",
    "        'L keras'          : [False],\n",
    "    }\n",
    "        }\n",
    "\n",
    "multi_grid_search(dct1,True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part2/Task2/1Layer/CostLayer1Units.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part2/Task2/1Layer/AccuracyLayer1Units.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part2/Task2/1Layer/F1MacroLayer1Units.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part2/Task2/1Layer/F1MicroLayer1Units.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "<h5>SubTask 2 - Changing Number of Units in 2<sup>nd</sup> Hidden Layer of Architecture</h5>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dct2 = {\n",
    "    '2 Layer' :\n",
    "    {\n",
    "        'A datasets'       : ['Cat-Dog'],\n",
    "        'B units'          : it.product((1024,),(64,128,256)),\n",
    "        'C functions'      : it.product(('relu',),('relu',)),\n",
    "        'D preproc'        : ['standard'],\n",
    "        'E He'             : [True],\n",
    "        'F initmodes'      : ['gaussian'],\n",
    "        'G epochs'         : [100],\n",
    "        'H batchsize'      : [200],\n",
    "        'I learning_rate'  : [0.0001],\n",
    "        'J optimizer'      : ['myopt'],\n",
    "        'K early_stopping' : [(True,True,-1,True)],\n",
    "        'L keras'          : [False],\n",
    "    }\n",
    "        }\n",
    "\n",
    "multi_grid_search(dct2,True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part2/Task2/2Layer/CostLayer2Units.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part2/Task2/2Layer/AccuracyLayer2Units.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part2/Task2/2Layer/F1MacroLayer2Units.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part2/Task2/2Layer/F1MicroLayer2Units.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "<h5>SubTask 3 - Changing Number of Units in 3<sup>rd</sup> Hidden Layer of Architecture</h5>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dct3 = {\n",
    "    '3 Layer' :\n",
    "    {\n",
    "        'A datasets'       : ['Cat-Dog'],\n",
    "        'B units'          : it.product((1024,),(256,),(16,32,64)),\n",
    "        'C functions'      : it.product(('relu',),('relu',),('tanh',)),\n",
    "        'D preproc'        : ['standard'],\n",
    "        'E He'             : [True],\n",
    "        'F initmodes'      : ['gaussian'],\n",
    "        'G epochs'         : [100],\n",
    "        'H batchsize'      : [200],\n",
    "        'I learning_rate'  : [0.0001],\n",
    "        'J optimizer'      : ['myopt'],\n",
    "        'K early_stopping' : [(True,True,-1,True)],\n",
    "        'L keras'          : [False],\n",
    "    }\n",
    "        }\n",
    "\n",
    "multi_grid_search(dct3,True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part2/Task2/3Layer/CostLayer3Units.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part2/Task2/3Layer/AccuracyLayer3Units.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part2/Task2/3Layer/F1MacroLayer3Units.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part2/Task2/3Layer/F1MicroLayer3Units.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "<h3>Task 3 - Trying Activation Functions on each layer</h3>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "<h5>SubTask 1 - Changing Activation Functions in 1<sup>st</sup> Hidden Layer of Architecture</h5>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dct1 =  {\n",
    "    '1 Layer FUNCTIONS' :\n",
    "    {\n",
    "        'A datasets'       : ['Cat-Dog'],\n",
    "        'B units'          : it.product((2048,)),\n",
    "        'C functions'      : it.product(('sigmoid','relu','tanh','swish')),\n",
    "        'D preproc'        : ['standard'],\n",
    "        'E He'             : [True],\n",
    "        'F initmodes'      : ['gaussian'],\n",
    "        'G epochs'         : [100],\n",
    "        'H batchsize'      : [200],\n",
    "        'I learning_rate'  : [0.0001],\n",
    "        'J optimizer'      : ['myopt'],\n",
    "        'K early_stopping' : [(True,True,-1,True)],\n",
    "        'L keras'          : [False],\n",
    "    }\n",
    "        }\n",
    "\n",
    "multi_grid_search(dct1,True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part2/Task3/1LayerF/CostLayer1Activation.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part2/Task3/1LayerF/AccuracyLayer1Activation.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part2/Task3/1LayerF/F1MacroLayer1Activation.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part2/Task3/1LayerF/F1MicroLayer1Activation.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "<h5>SubTask 2 - Changing Activation Functions in 2<sup>nd</sup> Hidden Layer of Architecture</h5>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dct2 = {\n",
    "    '2 Layer FUNCTIONS' :\n",
    "    {\n",
    "        'A datasets'       : ['Cat-Dog'],\n",
    "        'B units'          : it.product((1024,),(256,)),\n",
    "        'C functions'      : it.product(('relu',),('sigmoid','relu','tanh','swish')),\n",
    "        'D preproc'        : ['standard'],\n",
    "        'E He'             : [True],\n",
    "        'F initmodes'      : ['gaussian'],\n",
    "        'G epochs'         : [100],\n",
    "        'H batchsize'      : [200],\n",
    "        'I learning_rate'  : [0.0001],\n",
    "        'J optimizer'      : ['myopt'],\n",
    "        'K early_stopping' : [(True,True,-1,True)],\n",
    "        'L keras'          : [False],\n",
    "    }\n",
    "        }\n",
    "\n",
    "multi_grid_search(dct2,True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part2/Task3/2LayerF/CostLayer2Activation.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part2/Task3/2LayerF/AccuracyLayer2Activation.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part2/Task3/2LayerF/F1MacroLayer2Activation.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part2/Task3/2LayerF/F1MicroLayer2Activation.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "<h5>SubTask 3 - Changing Activation Functions in 3<sup>rd</sup> Hidden Layer of Architecture</h5>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dct3 = {\n",
    "    '3 Layer FUNCTIONS' :\n",
    "    {\n",
    "        'A datasets'       : ['Cat-Dog'],\n",
    "        'B units'          : it.product((1024,),(256,),(64,)),\n",
    "        'C functions'      : it.product(('relu',),('relu',),('relu','tanh','swish'),),\n",
    "        'D preproc'        : ['standard'],\n",
    "        'E He'             : [True],\n",
    "        'F initmodes'      : ['gaussian'],\n",
    "        'G epochs'         : [100],\n",
    "        'H batchsize'      : [200],\n",
    "        'I learning_rate'  : [0.0001],\n",
    "        'J optimizer'      : ['myopt'],\n",
    "        'K early_stopping' : [(True,True,-1,True)],\n",
    "        'L keras'          : [False],\n",
    "    }\n",
    "        }\n",
    "\n",
    "multi_grid_search(dct3,True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part2/Task3/3LayerF/CostLayer3Activation.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part2/Task3/3LayerF/AccuracyLayer3Activation.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part2/Task3/3LayerF/F1MacroLayer3Activation.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part2/Task3/3LayerF/F1MicroLayer3Activation.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Task 4 Initialization & Preprocessing Techniques</h3>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "<h5>SubTask 1 Impact of Xavier-He weight Initiliazation</h5>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dct2 =  {\n",
    "    'Xavier-He':\n",
    "    {\n",
    "        'A datasets'       : ['Cat-Dog'],\n",
    "        'B units'          : it.product((1024,),(256,)),\n",
    "        'C functions'      : it.product(('relu',),('relu',)),\n",
    "        'D preproc'        : ['standard'],\n",
    "        'E He'             : [False,True],\n",
    "        'F initmodes'      : ['gaussian'],\n",
    "        'G epochs'         : [100],\n",
    "        'H batchsize'      : [200],\n",
    "        'I learning_rate'  : [0.0001],\n",
    "        'J optimizer'      : ['myopt'],\n",
    "        'K early_stopping' : [(True,True,-1,True)],\n",
    "        'L keras'          : [False],\n",
    "    }\n",
    "        }\n",
    "\n",
    "multi_grid_search(dct2,True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h5>Observed learning curve for Initialization technique as \"Default\" vs \"Xavier-He\"</h5>\n",
    "<br><img src=\"output_plots/Part2/Task4/XH/NHE.png\" height=\"300\" width=\"450\" align=\"left\">\n",
    "<img src=\"output_plots/Part2/Task4/XH/YHE.png\" height=\"300\" width=\"450\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "<h5>SubTask 2 Finding Suitable Preprocesing & Initialization Distirbution</h5>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dct1 =  {\n",
    "    'Init Gaussian':\n",
    "    {\n",
    "    'A datasets'       : ['Cat-Dog'],\n",
    "    'B units'          : it.product((1024,),(256,)),\n",
    "    'C functions'      : it.product(('relu',),('relu',)),\n",
    "    'D preproc'        : ['standard','scale'],\n",
    "    'E He'             : [True],\n",
    "    'F initmodes'      : ['gaussian'],\n",
    "    'G epochs'         : [100],\n",
    "    'H batchsize'      : [200],\n",
    "    'I learning_rate'  : [0.0001],\n",
    "    'J optimizer'      : ['myopt'],\n",
    "    'K early_stopping' : [(True,True,-1,True)],\n",
    "    'L keras'          : [False],\n",
    "    },\n",
    "\n",
    "    'Init Uniform':\n",
    "    {\n",
    "    'A datasets'       : ['Cat-Dog'],\n",
    "    'B units'          : it.product((1024,),(256,)),\n",
    "    'C functions'      : it.product(('relu',),('relu',)),\n",
    "    'D preproc'        : ['standard','scale'],\n",
    "    'E He'             : [True],\n",
    "    'F initmodes'      : ['uniform'],\n",
    "    'G epochs'         : [100],\n",
    "    'H batchsize'      : [200],\n",
    "    'I learning_rate'  : [0.0001],\n",
    "    'J optimizer'      : ['myopt'],\n",
    "    'K early_stopping' : [(True,True,-1,True)],\n",
    "    'L keras'          : [False],\n",
    "    }\n",
    "        }\n",
    "\n",
    "multi_grid_search(dct1,True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "<h5>Observed learning curve for Preprocessing from \"Scaling\" vs \"Standardization\"</h5>\n",
    "<br><img src=\"output_plots/Part2/Task4/IG/GraphG2.png\" height=\"300\" width=\"450\" align=\"left\">\n",
    "<img src=\"output_plots/Part2/Task4/IG/GraphG.png\" height=\"250\" width=\"450\" align=\"right\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "<b>Note: </b>Standard preprocessing, being insensitive to out-liers performs better than Min-Max Scaling<br>\n",
    "Hence, most our experiments shown use standardization, and might follow same for future ones"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "<h5>Observed learning curve for Initialization from \"Uniform\" vs \"Gaussian\"</h5>\n",
    "<br><img src=\"output_plots/Part2/Task4/IU/GraphU.png\" height=\"300\" width=\"450\" align=\"left\">\n",
    "<img src=\"output_plots/Part2/Task4/IG/GraphG.png\" height=\"250\" width=\"450\" align=\"right\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>General obervation:</b>\n",
    "Initialization when done from Gaussian distribution inserts minimum information in a system<br>\n",
    "Hence our most experiments will use it, you are free to experiment with other options"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Task 5 Comparing Classification Reports of Numpy & Keras implementations</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dct1 =  {\n",
    "    'Keras':\n",
    "    {\n",
    "        'A datasets'       : ['Cat-Dog'],\n",
    "        'B units'          : it.product((2048,),(256,),(64,)),\n",
    "        'C functions'      : it.product(('relu',),('relu',),('tanh',)),\n",
    "        'D preproc'        : ['standard'],\n",
    "        'E He'             : [True],\n",
    "        'F initmodes'      : ['gaussian'],\n",
    "        'G epochs'         : [100],\n",
    "        'H batchsize'      : [200],\n",
    "        'I learning_rate'  : [0.0001],\n",
    "        'J optimizer'      : ['myopt'],\n",
    "        'K early_stopping' : [(True,True,-1,True)],\n",
    "        'L keras'          : [True],\n",
    "    },\n",
    "        }\n",
    "\n",
    "multi_grid_search(dct1,True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<pre>\n",
    "\n",
    "\t\t ################ NumPy Implementation ################ \n",
    "\n",
    "        -------- Classification Report on Training data -------- \n",
    "               precision    recall  f1-score   support\n",
    "\n",
    "         cat     1.0000    1.0000    1.0000     11250\n",
    "         dog     1.0000    1.0000    1.0000     11250\n",
    "\n",
    "   micro avg     1.0000    1.0000    1.0000     22500\n",
    "   macro avg     1.0000    1.0000    1.0000     22500\n",
    "weighted avg     1.0000    1.0000    1.0000     22500\n",
    " samples avg     1.0000    1.0000    1.0000     22500\n",
    "\n",
    "\n",
    "\t\t -------- Classification Report on Validation data -------- \n",
    "               precision    recall  f1-score   support\n",
    "\n",
    "         cat     0.6341    0.6392    0.6367      1250\n",
    "         dog     0.6363    0.6312    0.6337      1250\n",
    "\n",
    "   micro avg     0.6352    0.6352    0.6352      2500\n",
    "   macro avg     0.6352    0.6352    0.6352      2500\n",
    "weighted avg     0.6352    0.6352    0.6352      2500\n",
    " samples avg     0.6352    0.6352    0.6352      2500\n",
    "\n",
    "\n",
    "\n",
    "\t\t ################ Keras Implementation ################ \n",
    "         \n",
    "         -------- Classification Report on Training data -------- \n",
    "               precision    recall  f1-score   support\n",
    "\n",
    "         cat     0.9932    0.9986    0.9959     11250\n",
    "         dog     0.9986    0.9932    0.9959     11250\n",
    "\n",
    "   micro avg     0.9959    0.9959    0.9959     22500\n",
    "   macro avg     0.9959    0.9959    0.9959     22500\n",
    "weighted avg     0.9959    0.9959    0.9959     22500\n",
    " samples avg     0.9959    0.9959    0.9959     22500\n",
    "\n",
    "\n",
    "\t\t -------- Classification Report on Validation data -------- \n",
    "               precision    recall  f1-score   support\n",
    "\n",
    "         cat     0.6193    0.7224    0.6669      1250\n",
    "         dog     0.6670    0.5560    0.6065      1250\n",
    "\n",
    "   micro avg     0.6392    0.6392    0.6392      2500\n",
    "   macro avg     0.6432    0.6392    0.6367      2500\n",
    "weighted avg     0.6432    0.6392    0.6367      2500\n",
    " samples avg     0.6392    0.6392    0.6392      2500"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<pre>\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "<h1>Part 3 Execution of Neural Nets on Datasets of Assignment 1</h1>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "<h3>Loading Datasets & Preprocessing of Twitter data into bag-of-word</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# 1. Dolphins\n",
    "X1 = pd.read_csv('/data2/dolphins/dolphins.csv',sep=' ',header=None)\n",
    "Y1 = pd.read_csv('/data2/dolphins/dolphins_label.csv',sep=' ',header=None)\n",
    "X1, Y1 = np.array(X1), np.array(Y1)\n",
    "Y1 = cattooht(Y1)\n",
    "\n",
    "# 2. Twitter(bag-of-Word)\n",
    "X2 = pd.read_csv('/data2/twitter/twitter.csv',header=None)\n",
    "Y2 = pd.read_csv('/data2/twitter/twitter_label.csv',header=None)\n",
    "\n",
    "# Converting into bag-of-word\n",
    "all_words = set()\n",
    "local_ls = []\n",
    "for indx,stmt in X2.iterrows():\n",
    "    local = {}\n",
    "    for word in stmt[0].strip().split():\n",
    "        if word in local:\n",
    "            local[word] += 1\n",
    "        else:\n",
    "            local[word]  = 1\n",
    "        \n",
    "    local_ls.append(local)\n",
    "    all_words.update(local)\n",
    "\n",
    "mat = [[(local[word] if word in local else 0) for word in all_words] for local in local_ls]\n",
    "X2 = pd.DataFrame(np.array(mat))\n",
    "X2, Y2 = np.array(X2), np.array(Y2)\n",
    "Y2 = cattooht(Y2)\n",
    "\n",
    "# 3. PubMed\n",
    "X3 = pd.read_csv('/data2/pubmed/pubmed.csv',sep=' ',header=None)\n",
    "Y3 = pd.read_csv('/data2/pubmed/pubmed_label.csv',sep=' ',header=None)\n",
    "X3, Y3 = np.array(X3), np.array(Y3)\n",
    "Y3 = cattooht(Y3)\n",
    "\n",
    "\n",
    "# Assignment into Global Datastructure\n",
    "datasets['Dolphins'] = X1,Y1[0],list(map(str,Y1[1]))\n",
    "datasets['Twitter']  = X2,Y2[0],list(map(str,Y2[1]))\n",
    "datasets['PubMed']   = X3,Y3[0],list(map(str,Y3[1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "<h3> Evaluation on Dolphins datasets</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1. Dolphins Execution\n",
    "name = 'Dolphins'\n",
    "ipdim, opdim = datasets[name][0].shape[1], datasets[name][1].shape[1]\n",
    "s1 = ipdim*2\n",
    "s2 = int(round((s1*opdim)**0.5))\n",
    "name,config = name,[(s1,'relu'),(s2,'relu')]\n",
    "ln = len(datasets[name][0])\n",
    "_ = evaldata(name,config,'standard',True,'gaussian',100,10,0.001,'myopt',(True,True,-1,True),False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<pre>Previous Bayesian Classification Metrics\n",
    "Accuracy : 0.90833   F1-Micro : 0.90833    F1-Macro : 0.85333\n",
    "\n",
    "Previous KNN Classification Metrices\n",
    "Accuracy  0.98333    F1-Micro : 0.98333    F1-Macro : 0.98222"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"output_plots/Part3/Dolphins.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>General obervation:</b>\n",
    "Neural Network is able to make precise prediction in class imbalanced dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "<h3> Evaluation on Twitter datasets</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "name = 'Twitter'\n",
    "ipdim, opdim = datasets[name][0].shape[1], datasets[name][1].shape[1]\n",
    "s1 = ipdim*2\n",
    "s2 = int(round((s1*opdim)**0.5))\n",
    "name,config = name,[(s1,'relu'),(s2,'relu')]\n",
    "ln = len(datasets[name][0])\n",
    "_ = evaldata(name,config,'standard',True,'gaussian',10,300,0.0003,'myopt',(True,True,-1,True),False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<pre>Previous Bayesian Classification Metrics\n",
    "Accuracy : 0.56010   F1-Micro : 0.56010    F1-Macro : 0.34854\n",
    "\n",
    "Previous KNN Classification Metrices\n",
    "Accuracy  0.48414    F1-Micro : 0.48414    F1-Macro : 0.41388"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "<img src=\"output_plots/Part3/Twittr.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>General obervation:</b>\n",
    "No improvement on validation data is observed during training<br>\n",
    "Network is just overfitting Training data, sequence modelling should be captured<br>\n",
    "Which is not possible with ordinary Neural Network Architecture of this assignment"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "<h3> Evaluation on PubMed datasets</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "name = 'PubMed'\n",
    "ipdim, opdim = datasets[name][0].shape[1], datasets[name][1].shape[1]\n",
    "s1 = ipdim*2\n",
    "s2 = int(round((s1*opdim)**0.5))\n",
    "name,config = name,[(s1,'relu'),(s2,'relu')]\n",
    "ln = len(datasets[name][0])\n",
    "_ = evaldata(name,config,'standard',True,'gaussian',100,500,0.001,'myopt',(True,True,-1,True),False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<pre>Previous Bayesian Classification Metrics\n",
    "Accuracy : 0.44144   F1-Micro : 0.44144    F1-Macro : 0.33277\n",
    "\n",
    "Previous KNN Classification Metrices\n",
    "Accuracy  0.35412    F1-Micro : 0.35412    F1-Macro : 0.34554"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "<img src=\"output_plots/Part3/PubMed.png\" height=\"450\" width=\"600\" align=\"left\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>General obervation:</b>\n",
    "This dataset is again overfitted by Neural Network<br>\n",
    "Multinomial Naive Bayes' performed some better than Neural network"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python3(DSL)",
   "language": "python",
   "name": "dsl"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}