CNN.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/AaronThomasJoseph/DL-Projects/blob/main/CNN.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "w7Mdg7VpphbK"
      },
      "outputs": [],
      "source": [
        "#Libraries\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "import zipfile\n",
        "import os\n",
        "import numpy as np\n",
        "import tensorflow\n",
        "import tensorflow as tf\n",
        "import keras\n",
        "from tensorflow.keras.applications.vgg19 import VGG19\n",
        "from keras.applications.vgg19 import preprocess_input, decode_predictions\n",
        "from keras import datasets, layers, models\n",
        "from keras.preprocessing import image\n",
        "from keras.models import Model\n",
        "from keras.layers import Input, Dense, Flatten, GlobalAveragePooling2D\n",
        "from keras.preprocessing.image import ImageDataGenerator\n",
        "from keras.callbacks import EarlyStopping\n",
        "from keras.utils.vis_utils import plot_model\n",
        "from sklearn.preprocessing import OneHotEncoder\n",
        "from sklearn.metrics import confusion_matrix\n",
        "from sklearn.metrics import accuracy_score\n",
        "from sklearn.metrics import precision_score\n",
        "from sklearn.metrics import recall_score\n",
        "from sklearn.metrics import f1_score\n",
        "from sklearn.utils import shuffle\n",
        "from scipy import signal"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GrlnEfKup-ld"
      },
      "outputs": [],
      "source": [
        "#Loading Image Data\n",
        "with zipfile.ZipFile(\"Group21_Assignment3.zip\",\"r\") as zip_ref:\n",
        "    zip_ref.extractall()\n",
        "zip_ref.close()\n",
        "\n",
        "def load_images_from_folder(folder):\n",
        "    images = []\n",
        "    for filename in os.listdir(folder):\n",
        "        img = plt.imread(os.path.join(folder,filename))\n",
        "        dim = img.shape\n",
        "        if len(dim)!=3: #Grayscale image\n",
        "          img = np.array([img.T,img.T,img.T]).T\n",
        "\n",
        "        img   = tensorflow.image.resize(img,size = (224,224),\n",
        "        method=tensorflow.image.ResizeMethod.BILINEAR,preserve_aspect_ratio=False)\n",
        "\n",
        "        if img is not None:\n",
        "            images.append(np.array(img))\n",
        "    return images"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5GfMyWQcqXCF"
      },
      "source": [
        "#**`Dataset`**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rmzSkTY4qWlp"
      },
      "outputs": [],
      "source": [
        "#Train Data\n",
        "train_butterfly = np.array(load_images_from_folder(\"Group_21/train/butterfly\"))\n",
        "train_kangaroo  = np.array(load_images_from_folder(\"Group_21/train/kangaroo\"))\n",
        "train_sunflower = np.array(load_images_from_folder(\"Group_21/train/sunflower\"))\n",
        "\n",
        "x_train = np.concatenate([train_butterfly,train_kangaroo,train_sunflower])\n",
        "y_train = np.concatenate([0+np.zeros(len(train_butterfly)),1+np.zeros(len(train_kangaroo)),\n",
        "                          2+np.zeros(len(train_sunflower))])\n",
        "\n",
        "#Validation Data\n",
        "val_butterfly = np.array(load_images_from_folder(\"Group_21/val/butterfly\"))\n",
        "val_kangaroo  = np.array(load_images_from_folder(\"Group_21/val/kangaroo\"))\n",
        "val_sunflower = np.array(load_images_from_folder(\"Group_21/val/sunflower\"))\n",
        "\n",
        "x_val = np.concatenate([val_butterfly,val_kangaroo,val_sunflower])\n",
        "y_val = np.concatenate([0+np.zeros(len(val_butterfly)),1+np.zeros(len(val_kangaroo)),\n",
        "                          2+np.zeros(len(val_sunflower))])\n",
        "\n",
        "#Test Data\n",
        "test_butterfly = np.array(load_images_from_folder(\"Group_21/test/butterfly\"))\n",
        "test_kangaroo  = np.array(load_images_from_folder(\"Group_21/test/kangaroo\"))\n",
        "test_sunflower = np.array(load_images_from_folder(\"Group_21/test/sunflower\"))\n",
        "\n",
        "x_test = np.concatenate([test_butterfly,test_kangaroo,test_sunflower])\n",
        "y_test = np.concatenate([0+np.zeros(len(test_butterfly)),1+np.zeros(len(test_kangaroo)),\n",
        "                          2+np.zeros(len(test_sunflower))])\n",
        "\n",
        "del train_butterfly,train_kangaroo,train_sunflower\n",
        "del val_butterfly,val_kangaroo,val_sunflower\n",
        "del test_butterfly,test_kangaroo,test_sunflower\n",
        "\n",
        "#Normalizing Input\n",
        "x_train,x_val,x_test = x_train/255,x_val/255,x_test/255"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "E9ZAC-RdHnMI"
      },
      "source": [
        "# **`Q1`**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mefONErgqvfu"
      },
      "outputs": [],
      "source": [
        "np.random.seed(2500)\n",
        "for i in set(y_train):\n",
        "  index = list(range(len(y_train[y_train==i])))\n",
        "  np.random.shuffle(index)\n",
        "img_butterfly_colour = x_train[y_train==0][index[0]]\n",
        "img_kangaroo_colour  = x_train[y_train==1][index[0]]\n",
        "img_sunflower_colour = x_train[y_train==2][index[0]]\n",
        "\n",
        "#Grayscale\n",
        "def Colour_to_GrayScale(IMG):\n",
        "  return (0.2989*IMG.T[0] + 0.5870*IMG.T[1] + 0.1140*IMG.T[2]).T\n",
        "\n",
        "img_butterfly = Colour_to_GrayScale(img_butterfly_colour)\n",
        "img_kangaroo  = Colour_to_GrayScale(img_kangaroo_colour)\n",
        "img_sunflower = Colour_to_GrayScale(img_sunflower_colour)\n",
        "\n",
        "def Plot_Image(nrows,ncols,index,IMG,title,gray=True):\n",
        "  #IMG = np.clip(a = IMG, a_min = 0, a_max = 1)\n",
        "  plt.subplot(nrows,ncols,index)\n",
        "  if gray:  plt.imshow(IMG,cmap='gray')\n",
        "  else:     plt.imshow(IMG)\n",
        "  plt.xticks([]);plt.yticks([])\n",
        "  plt.title(title)\n",
        "\n",
        "plt.figure(figsize=(8,8))\n",
        "Plot_Image(nrows=3,ncols=3,index=1,IMG=img_butterfly_colour,title='Butterfly\\nColour')\n",
        "Plot_Image(nrows=3,ncols=3,index=2,IMG=img_kangaroo_colour, title='Kangaroo\\nColour')\n",
        "Plot_Image(nrows=3,ncols=3,index=3,IMG=img_sunflower_colour,title='Sunflower\\nColour')\n",
        "\n",
        "Plot_Image(nrows=3,ncols=3,index=4,IMG=img_butterfly,title='Butterfly\\nGrayScale')\n",
        "Plot_Image(nrows=3,ncols=3,index=5,IMG=img_kangaroo, title='Kangaroo\\nGrayScale')\n",
        "Plot_Image(nrows=3,ncols=3,index=6,IMG=img_sunflower,title='Sunflower\\nGrayScale')\n",
        "\n",
        "convolutional_filter = np.random.normal(loc=0,scale=np.sqrt(2/3**2),size=(3,3))\n",
        "print(\"Convolutional Filter:\\n\",np.around(convolutional_filter,5))\n",
        "\n",
        "S = 1 #Stride\n",
        "P = 0 #Padding\n",
        "def Conv2D(a, f):\n",
        "    s = f.shape + tuple(np.subtract(a.shape, f.shape) + 1)\n",
        "    strd = np.lib.stride_tricks.as_strided\n",
        "    subM = strd(a, shape = s, strides = a.strides * 2)\n",
        "    return np.einsum('ij,ijkl->kl', f, subM)\n",
        "\n",
        "conv_butterfly = Conv2D(img_butterfly,convolutional_filter)\n",
        "conv_kangaroo  = Conv2D(img_kangaroo, convolutional_filter)\n",
        "conv_sunflower = Conv2D(img_sunflower,convolutional_filter)\n",
        "\n",
        "Plot_Image(nrows=3,ncols=3,index=7,IMG=conv_butterfly,title='Butterfly\\nFeature Map')\n",
        "Plot_Image(nrows=3,ncols=3,index=8,IMG=conv_kangaroo, title='Kangaroo\\nFeature Map')\n",
        "Plot_Image(nrows=3,ncols=3,index=9,IMG=conv_sunflower,title='Sunflower\\nFeature Map')\n",
        "plt.tight_layout()\n",
        "plt.show()\n",
        "\n",
        "W1,H1 = img_butterfly.shape\n",
        "F1,F2 = convolutional_filter.shape\n",
        "W2    = int((W1-F1+2*P)/S + 1)\n",
        "H2    = int((H1-F2+2*P)/S + 1)\n",
        "print(\"Expected dimension of feature map:\\t\",(W2,H2))\n",
        "print(\"Dimension of feature map obtained:\\t\",conv_butterfly.shape)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "44gHGPhmQw7Z"
      },
      "source": [
        "# **`Q2`**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hdwtyO4BQwZ8"
      },
      "outputs": [],
      "source": [
        "class Convolutional_Layer:\n",
        "  def __init__(self,Image,f1,f2):\n",
        "    self.S = 1 #Stride\n",
        "    self.P = 0 #Padding\n",
        "    self.Image = Image\n",
        "    #np.random.seed(50)\n",
        "    self.Filter1    =  np.random.normal(loc=0,scale=np.sqrt(2/3**2)     ,size=(f1,3,3))\n",
        "    self.Filter2    =  np.random.normal(loc=0,scale=np.sqrt(2/(3**2*32)),size=(f2,3,3))\n",
        "\n",
        "  def Conv2D(self,a, f):\n",
        "    s = f.shape + tuple(np.subtract(a.shape, f.shape) + 1)\n",
        "    strd = np.lib.stride_tricks.as_strided\n",
        "    subM = strd(a, shape = s, strides = a.strides * 2)\n",
        "    return np.einsum('ij,ijkl->kl', f, subM)\n",
        "\n",
        "  def ReLU(self,IMG):\n",
        "    IMG[IMG<0] = 0\n",
        "    return IMG\n",
        "\n",
        "  def Plot_Image(self,nrows,ncols,index,IMG,title):\n",
        "    #IMG = np.clip(a = IMG, a_min = 0, a_max = 1)\n",
        "    plt.subplot(nrows,ncols,index)\n",
        "    plt.imshow(IMG,cmap='gray')\n",
        "    plt.xticks([]);plt.yticks([])\n",
        "    plt.title(title)\n",
        "\n",
        "  def Layer1(self):\n",
        "    Image1 = []\n",
        "    for i in range(len(self.Filter1)):\n",
        "      Image1.append(self.Conv2D(self.Image,self.Filter1[i]))\n",
        "    Image1 = self.ReLU(np.array(Image1))\n",
        "\n",
        "    #Index = list(range(len(Image1)))\n",
        "    #np.random.shuffle(Index)\n",
        "    plt.figure(figsize=(8,8))\n",
        "    plt.suptitle(\"Layer 1\",fontweight=\"bold\",y=1+1e-2)\n",
        "    index = 1\n",
        "    for i in range(10):\n",
        "      self.Plot_Image(nrows=4,ncols=5,index=index,IMG=self.Filter1[i],title=\"Filter \"+str(i+1))\n",
        "      self.Plot_Image(nrows=4,ncols=5,index=index+10,IMG=Image1[i],title=\"Filter Map \"+str(i+1))\n",
        "      index+=1\n",
        "    plt.tight_layout()\n",
        "    plt.show()\n",
        "    return Image1\n",
        "\n",
        "  def Layer2(self):\n",
        "    Image1 = self.Layer1()\n",
        "\n",
        "    Image2 = []\n",
        "    for i in range(len(self.Filter2)):\n",
        "      Img = 0\n",
        "      for j in range(len(Image1)):\n",
        "        Img += self.Conv2D(Image1[j],self.Filter2[i])\n",
        "      Image2.append(Img)\n",
        "    Image2 = self.ReLU(np.array(Image2))\n",
        "\n",
        "    #Index = list(range(len(Image2)))\n",
        "    #np.random.shuffle(Index)\n",
        "    plt.figure(figsize=(8,8))\n",
        "    plt.suptitle(\"Layer 2\",fontweight=\"bold\",y=1+1e-2)\n",
        "    index = 1\n",
        "    for i in range(10):\n",
        "      self.Plot_Image(nrows=4,ncols=5,index=index,IMG=self.Filter2[i],title=\"Filter \"+str(i+1))\n",
        "      self.Plot_Image(nrows=4,ncols=5,index=index+10,IMG=Image2[i],title=\"Filter Map \"+str(i+1))\n",
        "      index+=1\n",
        "    plt.tight_layout()\n",
        "    plt.show()\n",
        "\n",
        "    print(\"Layer 1:\")\n",
        "    W1,H1 = self.Image.shape\n",
        "    F1,F2 = self.Filter1[0].shape\n",
        "    W2    = int((W1-F1+2*P)/S + 1)\n",
        "    H2    = int((H1-F2+2*P)/S + 1)\n",
        "    print(\"Expected dimension of feature map:\\t\",(W2,H2))\n",
        "    print(\"Dimension of feature map obtained:\\t\",Image1[0].shape)\n",
        "\n",
        "    print(\"\\nLayer 2:\")\n",
        "    W1,H1 = Image1[0].shape\n",
        "    F1,F2 = self.Filter2[0].shape\n",
        "    W2    = int((W1-F1+2*P)/S + 1)\n",
        "    H2    = int((H1-F2+2*P)/S + 1)\n",
        "    print(\"Expected dimension of feature map:\\t\",(W2,H2))\n",
        "    print(\"Dimension of feature map obtained:\\t\",Image2[0].shape)\n",
        "    #return Image2"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OS7qVAkFVE68"
      },
      "outputs": [],
      "source": [
        "print(\"Butterfly:\\n\")\n",
        "Butterfly = Convolutional_Layer(Image=img_butterfly,f1=32,f2=64)\n",
        "Butterfly.Layer2()\n",
        "\n",
        "print(\"\\nKangaroo:\\n\")\n",
        "Kangaroo = Convolutional_Layer(Image=img_kangaroo,f1=32,f2=64)\n",
        "Kangaroo.Layer2()\n",
        "\n",
        "print(\"\\nSunflower:\\n\")\n",
        "Sunflower = Convolutional_Layer(Image=img_sunflower,f1=32,f2=64)\n",
        "Sunflower.Layer2()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aU-y7nwaRlto"
      },
      "source": [
        "# **`Q3`**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9j7_lnleWj2u"
      },
      "outputs": [],
      "source": [
        "#Converting one hot encoded test label to label\n",
        "def ohetolabel(array):\n",
        "  output = list()\n",
        "  for i in range(len(array)):\n",
        "    output.append(np.argmax(array[i]))\n",
        "  return np.array(output)\n",
        "\n",
        "#Plot cf matrix\n",
        "def cf_matrix_plot(cf_matrix,title):\n",
        "  #group_names = ['True Neg','False Pos','False Neg','True Pos','True Pos','True Pos','True Pos','True Pos','True Pos']\n",
        "\n",
        "  group_counts = [\"{0:0.0f}\".format(value) for value in\n",
        "                  cf_matrix.flatten()]\n",
        "\n",
        "  group_percentages = [\"{0:.2%}\".format(value) for value in\n",
        "                      cf_matrix.flatten()/np.sum(cf_matrix)]\n",
        "\n",
        "  labels = [f\"{v1}\\n{v2}\\n\" for v1, v2 in\n",
        "            zip(group_counts,group_percentages)]\n",
        "\n",
        "  labels = np.asarray(labels).reshape(3,3)\n",
        "\n",
        "  ax = sns.heatmap(cf_matrix, annot=labels, fmt='', cmap='Blues')\n",
        "\n",
        "  ax.set_title(title);\n",
        "  ax.set_xlabel('\\nPredicted Image Class')\n",
        "  ax.set_ylabel('Actual Image Class');\n",
        "\n",
        "  ## Ticket labels - List must be in alphabetical order\n",
        "  ax.xaxis.set_ticklabels(['Butterfly','Kangaroo','Sunflower'])\n",
        "  ax.yaxis.set_ticklabels(['Butterfly','Kangaroo','Sunflower'])"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def CNN(x_train,x_val,x_test,y_train,y_val,y_test):\n",
        "  model = models.Sequential()\n",
        "  model.add(layers.Conv2D(32, (3, 3), activation='relu', input_shape=(224, 224, 3),kernel_initializer = tf.keras.initializers.HeNormal()))\n",
        "  model.add(layers.MaxPooling2D((2, 2)))\n",
        "  model.add(layers.Conv2D(64, (3, 3), activation='relu',kernel_initializer = tf.keras.initializers.HeNormal()))\n",
        "  model.add(layers.MaxPooling2D((2, 2)))\n",
        "\n",
        "\n",
        "  model.add(layers.Flatten())\n",
        "  model.add(layers.Dense(128, activation='relu',kernel_initializer = tf.keras.initializers.LecunNormal()))\n",
        "  model.add(layers.Dense(3,  activation='softmax',kernel_initializer = tf.keras.initializers.LecunNormal()))\n",
        "\n",
        "  tensorflow.keras.optimizers.SGD(learning_rate=0.01)\n",
        "  model.compile(optimizer='sgd',\n",
        "              loss='categorical_crossentropy',\n",
        "              metrics=['accuracy'])\n",
        "\n",
        "  ohe = OneHotEncoder()\n",
        "  y_train = ohe.fit_transform(y_train.reshape(-1,1)).toarray()\n",
        "  y_val = ohe.fit_transform(y_val.reshape(-1,1)).toarray()\n",
        "  y_test = ohe.fit_transform(y_test.reshape(-1,1)).toarray()\n",
        "\n",
        "  error_th = 1e-4\n",
        "  es = EarlyStopping(monitor='loss',mode='min',min_delta = error_th,verbose=0,patience=2)\n",
        "  history = model.fit(x_train, y_train, epochs=100,batch_size=1,verbose=0,\n",
        "                    validation_data=(x_val, y_val),callbacks=[es])\n",
        "\n",
        "  # this is for prediction purpose\n",
        "  y_train_pred = model.predict(x_train)\n",
        "  y_val_pred   = model.predict(x_val)\n",
        "  y_test_pred  = model.predict(x_test)\n",
        "\n",
        "  y_train      = ohetolabel(y_train)\n",
        "  y_val        = ohetolabel(y_val)\n",
        "  y_test       = ohetolabel(y_test)\n",
        "  y_train_pred = ohetolabel(y_train_pred)\n",
        "  y_val_pred   = ohetolabel(y_val_pred)\n",
        "  y_test_pred  = ohetolabel(y_test_pred)\n",
        "\n",
        "  print(\"No of Epochs =\",len(history.history[\"loss\"]))\n",
        "  plt.figure(figsize=(12.5,8))\n",
        "  plt.suptitle('CNN',fontweight=\"bold\",y=1+1e-2)\n",
        "  plt.subplot(1,2,2)\n",
        "  plt.plot(history.history['accuracy'],marker='.')\n",
        "  plt.plot(history.history['val_accuracy'],marker='.')\n",
        "  plt.title('Model accuracy')\n",
        "  plt.ylabel('Accuracy')\n",
        "  plt.xlabel('Epoch')\n",
        "  plt.legend(['Train', 'Validation'], loc='upper left')\n",
        "  plt.grid()\n",
        "\n",
        "  plt.subplot(1,2,1)\n",
        "  plt.plot(history.history['loss'],marker='.')\n",
        "  plt.plot(history.history['val_loss'],marker='.')\n",
        "  plt.title('Model loss')\n",
        "  plt.ylabel('Loss')\n",
        "  plt.xlabel('Epoch')\n",
        "  plt.legend(['Train', 'Validation'], loc='upper left')\n",
        "  plt.grid()\n",
        "  plt.tight_layout();plt.show()\n",
        "\n",
        "  print(\"Training Accuracy =\\t\",history.history['accuracy'][-1])\n",
        "  print(\"Validation Accuracy =\\t\",history.history['val_accuracy'][-1])\n",
        "\n",
        "  #Generate the confusion matrix\n",
        "  plt.figure(figsize=(12.5,4))\n",
        "  plt.suptitle('CNN',fontweight=\"bold\",y=1+1e-2)\n",
        "  plt.subplot(1,3,1)\n",
        "  cf_matrix = confusion_matrix(y_train, y_train_pred)\n",
        "  cf_matrix_plot(cf_matrix,\"Training Data\")\n",
        "\n",
        "  plt.subplot(1,3,2)\n",
        "  cf_matrix = confusion_matrix(y_val, y_val_pred)\n",
        "  cf_matrix_plot(cf_matrix,\"Validation Data\")\n",
        "\n",
        "  plt.subplot(1,3,3)\n",
        "  cf_matrix = confusion_matrix(y_test, y_test_pred)\n",
        "  cf_matrix_plot(cf_matrix,\"Test Data\")\n",
        "  plt.tight_layout()\n",
        "  plt.show()\n",
        "\n",
        "  print(\"\\tTraining Data\\t\\t\\t\\tValidation Data\\t\\t\\t\\tTest Data\")\n",
        "  print(\"Accuracy:\",end=\" \")\n",
        "  print(np.around(accuracy_score(y_train , y_train_pred),4),end=\"\\t\"*4)\n",
        "  print(np.around(accuracy_score(y_val , y_val_pred),4),end=\"\\t\"*5)\n",
        "  print(np.around(accuracy_score(y_test , y_test_pred),4))\n",
        "\n",
        "  print(\"Precision:\",end=\"\")\n",
        "  print(np.around(precision_score(y_train , y_train_pred,average=\"macro\"),4),end=\"\\t\"*4)\n",
        "  print(np.around(precision_score(y_val , y_val_pred,average=\"macro\"),4),end=\"\\t\"*5)\n",
        "  print(np.around(precision_score(y_test , y_test_pred,average=\"macro\"),4))\n",
        "\n",
        "  print(\"Recall:\",end=\"   \")\n",
        "  print(np.around(recall_score(y_train , y_train_pred,average=\"macro\"),4),end=\"\\t\"*4)\n",
        "  print(np.around(recall_score(y_val , y_val_pred,average=\"macro\"),4),end=\"\\t\"*5)\n",
        "  print(np.around(recall_score(y_test , y_test_pred,average=\"macro\"),4))\n",
        "\n",
        "  print(\"F1-Score:\",end=\" \")\n",
        "  print(np.around(f1_score(y_train , y_train_pred,average=\"macro\"),4),end=\"\\t\"*4)\n",
        "  print(np.around(f1_score(y_val , y_val_pred,average=\"macro\"),4),end=\"\\t\"*5)\n",
        "  print(np.around(f1_score(y_test , y_test_pred,average=\"macro\"),4))\n",
        "\n",
        "  return model"
      ],
      "metadata": {
        "id": "wrmum8qTq5l8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Z_yoFAf5Rwkv"
      },
      "outputs": [],
      "source": [
        "def one_hot_encoding(input):\n",
        "  output = []\n",
        "  for i in input:\n",
        "    tmp = [0,0,0]\n",
        "    tmp[int(i)] = 1\n",
        "    output.append(tmp)\n",
        "  return np.asarray(output).reshape(-1,3,1)\n",
        "\n",
        "def binary_cross_entropy(y_true, y_pred):\n",
        "    e = 1e-5\n",
        "    return np.mean(-y_true * np.log(y_pred + e) - (1 - y_true) * np.log(1 - y_pred + e))\n",
        "\n",
        "def binary_cross_entropy_prime(y_true, y_pred):\n",
        "    e = 1e-5\n",
        "    return ((1 - y_true) / (1 - y_pred + e) - y_true / (y_pred + e)) / np.size(y_true)\n",
        "\n",
        "def predict(network, input):\n",
        "    output = input\n",
        "    for layer in network:\n",
        "        output = layer.forward(output/1e0)\n",
        "        #print(layer,'\\t',output.max())\n",
        "    return output\n",
        "\n",
        "def Cnn(network, x_train, y_train, x_val, y_val, x_test, y_test, epochs = 1000, learning_rate = 0.01, verbose = True):\n",
        "    for e in range(epochs):\n",
        "        error = 0\n",
        "        y_train_pred = []\n",
        "        x_train, y_train = shuffle(x_train, y_train, random_state=0)\n",
        "        i = 0\n",
        "        for x, y in zip(x_train, y_train):\n",
        "            i+=1\n",
        "            # forward\n",
        "            output = predict(network, x)\n",
        "            y_train_pred.append(np.argmax(output))\n",
        "            print(i%len(y_train),y.reshape(3),'\\t',output.reshape(3))\n",
        "\n",
        "            # error\n",
        "            error += binary_cross_entropy(y, output)\n",
        "\n",
        "            # backward\n",
        "            grad = binary_cross_entropy_prime(y, output)\n",
        "            for layer in reversed(network):\n",
        "                grad = layer.backward(grad, learning_rate)\n",
        "\n",
        "\n",
        "        error /= len(x_train)\n",
        "\n",
        "        if verbose:\n",
        "            print(f\"{e + 1}/{epochs}, error:\\t{error}\",end='\\t\\t')\n",
        "            y_train_pred = np.array(y_train_pred)\n",
        "            print('acc:\\t',sum(np.argmax(y_train,axis=1).reshape(y_train_pred.shape)==y_train_pred)/len(y_train))\n",
        "\n",
        "    y_train_pred = []\n",
        "    y_val_pred   = []\n",
        "    y_test_pred  = []\n",
        "    for x, y in zip(x_train, y_train):\n",
        "        output = predict(network, x)\n",
        "        y_train_pred.append(np.argmax(output))\n",
        "    for x, y in zip(x_val, y_val):\n",
        "        output = predict(network, x)\n",
        "        y_val_pred.append(np.argmax(output))\n",
        "    for x, y in zip(x_test, y_test):\n",
        "        output = predict(network, x)\n",
        "        y_test_pred.append(np.argmax(output))\n",
        "\n",
        "    y_train = np.argmax(y_train,axis=1).reshape(-1)\n",
        "    y_val = np.argmax(y_val,axis=1).reshape(-1)\n",
        "    y_test = np.argmax(y_test,axis=1).reshape(-1)\n",
        "\n",
        "    #Generate the confusion matrix\n",
        "    plt.figure(figsize=(12.5,4))\n",
        "    plt.suptitle('VGG19',fontweight=\"bold\",y=1+1e-2)\n",
        "    plt.subplot(1,3,1)\n",
        "    cf_matrix = confusion_matrix(y_train, y_train_pred)\n",
        "    cf_matrix_plot(cf_matrix,\"Training Data\")\n",
        "\n",
        "    plt.subplot(1,3,2)\n",
        "    cf_matrix = confusion_matrix(y_val, y_val_pred)\n",
        "    cf_matrix_plot(cf_matrix,\"Validation Data\")\n",
        "\n",
        "    plt.subplot(1,3,3)\n",
        "    cf_matrix = confusion_matrix(y_test, y_test_pred)\n",
        "    cf_matrix_plot(cf_matrix,\"Test Data\")\n",
        "    plt.tight_layout()\n",
        "    plt.show()\n",
        "\n",
        "    print(\"\\tTraining Data\\t\\t\\t\\tValidation Data\\t\\t\\t\\tTest Data\")\n",
        "    print(\"Accuracy:\",end=\" \")\n",
        "    print(np.around(accuracy_score(y_train , y_train_pred),4),end=\"\\t\"*4)\n",
        "    print(np.around(accuracy_score(y_val , y_val_pred),4),end=\"\\t\"*5)\n",
        "    print(np.around(accuracy_score(y_test , y_test_pred),4))\n",
        "\n",
        "    print(\"Precision:\",end=\"\")\n",
        "    print(np.around(precision_score(y_train , y_train_pred,average=\"macro\"),4),end=\"\\t\"*4)\n",
        "    print(np.around(precision_score(y_val , y_val_pred,average=\"macro\"),4),end=\"\\t\"*5)\n",
        "    print(np.around(precision_score(y_test , y_test_pred,average=\"macro\"),4))\n",
        "\n",
        "    print(\"Recall:\",end=\"   \")\n",
        "    print(np.around(recall_score(y_train , y_train_pred,average=\"macro\"),4),end=\"\\t\"*4)\n",
        "    print(np.around(recall_score(y_val , y_val_pred,average=\"macro\"),4),end=\"\\t\"*5)\n",
        "    print(np.around(recall_score(y_test , y_test_pred,average=\"macro\"),4))\n",
        "\n",
        "    print(\"F1-Score:\",end=\" \")\n",
        "    print(np.around(f1_score(y_train , y_train_pred,average=\"macro\"),4),end=\"\\t\"*4)\n",
        "    print(np.around(f1_score(y_val , y_val_pred,average=\"macro\"),4),end=\"\\t\"*5)\n",
        "    print(np.around(f1_score(y_test , y_test_pred,average=\"macro\"),4))\n",
        "\n",
        "class Reshape_Layer:\n",
        "    def __init__(self, input_shape, output_shape):\n",
        "        self.input_shape = input_shape\n",
        "        self.output_shape = output_shape\n",
        "\n",
        "    def forward(self, input):\n",
        "        return np.reshape(input/1e3, self.output_shape)\n",
        "\n",
        "    def backward(self, output_gradient, learning_rate):\n",
        "        return np.reshape(output_gradient, self.input_shape)\n",
        "\n",
        "\n",
        "class Relu_Layer:\n",
        "    def __init__(self):\n",
        "        pass\n",
        "\n",
        "    def forward(self, input):\n",
        "        self.input = input\n",
        "        return np.maximum(self.input,0)\n",
        "\n",
        "    def backward(self, output_gradient, learning_rate):\n",
        "\n",
        "        tmp = self.input > 0\n",
        "        return np.multiply(output_gradient, tmp)\n",
        "\n",
        "class Softmax_Layer:\n",
        "\n",
        "    def forward(self, input):\n",
        "        tmp = np.exp(input-max(input))\n",
        "        self.output = tmp / np.sum(tmp)\n",
        "        return self.output\n",
        "\n",
        "    def backward(self, output_gradient, learning_rate):\n",
        "        n = np.size(self.output)\n",
        "        return np.dot((np.identity(n) - self.output.T) * self.output, output_gradient)\n",
        "\n",
        "class Dense_Layer:\n",
        "    def __init__(self, input_size, output_size):\n",
        "        self.weights = np.random.normal(loc=0.0, scale=np.sqrt(1/input_size), size=(output_size, input_size))\n",
        "        self.bias    = np.random.normal(loc=0.0, scale=np.sqrt(1/input_size), size=(output_size, 1))\n",
        "\n",
        "    def forward(self, input):\n",
        "        self.input = input\n",
        "        return np.dot(self.weights, self.input) + self.bias\n",
        "\n",
        "    def backward(self, output_gradient, learning_rate):\n",
        "        weights_gradient = np.dot(output_gradient, self.input.T)\n",
        "        input_gradient = np.dot(self.weights.T, output_gradient)\n",
        "        self.weights -= learning_rate * weights_gradient\n",
        "        self.bias -= learning_rate * output_gradient\n",
        "        return input_gradient\n",
        "\n",
        "class MaxPool:\n",
        "    def iterate_regions(self, image):\n",
        "        _, h, w,  = image.shape\n",
        "\n",
        "        new_h = h // 2\n",
        "        new_w = w // 2\n",
        "\n",
        "        for i in range(new_h):\n",
        "            for j in range(new_w):\n",
        "                im_region = image[:,(i*2):(i*2+2), (j*2):(j*2+2)]\n",
        "                yield im_region, i, j\n",
        "\n",
        "    def forward(self, input):\n",
        "\n",
        "        self.last_input = input\n",
        "        num_filters, h, w, = input.shape\n",
        "        output = np.zeros((num_filters, h//2, w//2))\n",
        "\n",
        "        for im_region, i, j in self.iterate_regions(input):\n",
        "            output[:,i,j] = np.amax(im_region.T,axis=(0,1))\n",
        "        return output\n",
        "\n",
        "    def backward(self, d_l_d_out, learning_rate):\n",
        "\n",
        "        d_l_d_input = np.zeros(self.last_input.shape)\n",
        "\n",
        "        for im_region, i, j in self.iterate_regions(self.last_input):\n",
        "            f, h, w = im_region.shape\n",
        "            amax = np.amax(im_region.T, axis=(0,1))\n",
        "            for i2 in range(h):\n",
        "                for j2 in range(w):\n",
        "                    for f2 in range(f):\n",
        "                        #if the pixel was the max value, copy the gradient to it\n",
        "                        if(im_region[f2,i2,j2] == amax[f2]):\n",
        "                            d_l_d_input[f2, i*2+i2, j*2+j2] = d_l_d_out[f2, i, j]\n",
        "                            break;\n",
        "        return d_l_d_input\n",
        "\n",
        "class Convolutional_Layer:\n",
        "    def __init__(self, input_shape, kernel_size, depth):\n",
        "        input_depth, input_height, input_width = input_shape\n",
        "        self.depth = depth\n",
        "        self.input_shape = input_shape\n",
        "        self.input_depth = input_depth\n",
        "        self.output_shape = (depth, input_height - kernel_size + 1, input_width - kernel_size + 1)\n",
        "        self.kernels_shape = (depth, input_depth, kernel_size, kernel_size)\n",
        "        self.kernels = np.random.normal(loc=0.0, scale=np.sqrt(2/self.kernels_shape[0]*self.kernels_shape[1]*self.kernels_shape[2]), size=self.kernels_shape)\n",
        "        self.biases  = np.random.normal(loc=0.0, scale=np.sqrt(2/self.kernels_shape[0]*self.kernels_shape[1]*self.kernels_shape[2]), size=self.output_shape)\n",
        "\n",
        "    def forward(self, input):\n",
        "        self.input = input\n",
        "        self.output = np.copy(self.biases)\n",
        "\n",
        "        for i in range(self.depth):\n",
        "            for j in range(self.input_depth):\n",
        "              self.output[i] += signal.correlate2d(self.input[j], self.kernels[i, j], \"valid\")\n",
        "        self.output = self.output\n",
        "        return self.output\n",
        "\n",
        "    def backward(self, output_gradient, learning_rate):\n",
        "        kernels_gradient = np.zeros(self.kernels_shape)\n",
        "        input_gradient   = np.zeros(self.input_shape)\n",
        "\n",
        "        for i in range(self.depth):\n",
        "            for j in range(self.input_depth):\n",
        "\n",
        "                kernels_gradient[i, j] = signal.correlate2d(self.input[j],output_gradient[i], \"valid\")\n",
        "                input_gradient[j] += signal.convolve2d(output_gradient[i], self.kernels[i, j], \"full\")\n",
        "\n",
        "        self.kernels -= learning_rate * kernels_gradient\n",
        "        self.biases -= learning_rate * output_gradient\n",
        "        return input_gradient\n",
        "\n",
        "\n",
        "# neural network\n",
        "network = [\n",
        "    Convolutional_Layer(input_shape = (3, 224, 224), kernel_size = 3, depth = 32),\n",
        "    Relu_Layer(),\n",
        "    MaxPool(),\n",
        "\n",
        "    Convolutional_Layer(input_shape = (32, 111, 111), kernel_size = 3, depth = 64),\n",
        "    Relu_Layer(),\n",
        "    MaxPool(),\n",
        "\n",
        "    Reshape_Layer(input_shape = (64, 54, 54), output_shape = (64 * 54 * 54, 1)),\n",
        "    Dense_Layer(input_size = 64 * 54 * 54, output_size = 128),\n",
        "    Relu_Layer(),\n",
        "\n",
        "    Dense_Layer(input_size = 128, output_size = 3),\n",
        "    Softmax_Layer()\n",
        "]"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "x_train = x_train.reshape(-1,3,224,224)\n",
        "x_val = x_val.reshape(-1,3,224,224)\n",
        "x_test = x_test.reshape(-1,3,224,224)\n",
        "\n",
        "cnn = Cnn(network,\n",
        "          x_train.reshape(-1,3,224,224), one_hot_encoding(y_train),\n",
        "          x_val.reshape(-1,3,224,224), one_hot_encoding(y_val),\n",
        "          x_test.reshape(-1,3,224,224), one_hot_encoding(y_test),\n",
        "          epochs = 25, learning_rate = 0.01, verbose = True)"
      ],
      "metadata": {
        "id": "vfasC-1sgQCf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "cnn = CNN(x_train,x_val,x_test,y_train,y_val,y_test)"
      ],
      "metadata": {
        "id": "GctHX8vGq-dY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def PlotFilters(cnn,input):\n",
        "\n",
        "  def Plot_Image(nrows,ncols,index,IMG,title,gray=True):\n",
        "    IMG = np.clip(a = IMG, a_min = 0, a_max = 1)\n",
        "    plt.subplot(nrows,ncols,index)\n",
        "    if gray:  plt.imshow(IMG,cmap='gray')\n",
        "    else:     plt.imshow(IMG)\n",
        "    plt.xticks([]);plt.yticks([])\n",
        "    plt.title(title)\n",
        "\n",
        "  filter1 = cnn.layers[0].get_weights()[0]\n",
        "  CL1_model = Model(inputs=cnn.input, outputs=cnn.layers[0].output)\n",
        "  CL1_output = CL1_model.predict(input.reshape(-1,224,224,3))\n",
        "  plt.figure(figsize=(10,10))\n",
        "  plt.suptitle(\"Layer 1\",fontweight=\"bold\",y=1+1e-2)\n",
        "  for i in range(10):\n",
        "    IMG = filter1[:,:,:,i]\n",
        "    Plot_Image(4,5,i+1,IMG,title='Filter '+str(i+1))\n",
        "    Plot_Image(4,5,i+11,CL1_output[:,:,:,i].reshape(222,222),title='Feature Map '+str(i+1))\n",
        "  plt.tight_layout()\n",
        "  plt.show()\n",
        "\n",
        "  filter2 = cnn.layers[2].get_weights()[0]\n",
        "  CL2_model = Model(inputs=cnn.input, outputs=cnn.layers[2].output)\n",
        "  CL2_output = CL2_model.predict(input.reshape(-1,224,224,3))\n",
        "  plt.figure(figsize=(10,10))\n",
        "  plt.suptitle(\"Layer 2\",fontweight=\"bold\",y=1+1e-2)\n",
        "  for i in range(10):\n",
        "    IMG = filter2[:,:,:,i]\n",
        "    Plot_Image(4,5,i+1,IMG[:,:,:3],title='Filter '+str(i+1))\n",
        "    Plot_Image(4,5,i+11,CL2_output[:,:,:,i].reshape(109,109),title='Feature Map '+str(i+1))\n",
        "  plt.tight_layout()\n",
        "  plt.show()\n",
        "\n",
        "print(\"Butterfly:\")\n",
        "PlotFilters(cnn,img_butterfly_colour)\n",
        "\n",
        "print(\"\\nKangaroo:\")\n",
        "PlotFilters(cnn,img_kangaroo_colour)\n",
        "\n",
        "print(\"\\nSunflower:\")\n",
        "PlotFilters(cnn,img_sunflower_colour)"
      ],
      "metadata": {
        "id": "hXyEdmRTtOZl"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def Back_Prop(input_image,title):\n",
        "  input_image = tf.convert_to_tensor(input_image.reshape(-1,224,224,3))\n",
        "  base_model = cnn\n",
        "  test_model = Model(inputs=base_model.inputs, outputs=base_model.get_layer('max_pooling2d_1').output)\n",
        "\n",
        "  result = base_model(input_image)\n",
        "\n",
        "  @tf.custom_gradient\n",
        "  def backpropRelu(x):\n",
        "      def grad(dy):\n",
        "          return tf.cast(dy>0,tf.float32)  * tf.cast(x>0,tf.float32) * dy\n",
        "      return tf.nn.relu(x), grad\n",
        "\n",
        "  layer_dict = [layer for layer in test_model.layers[1:] if hasattr(layer,'activation')]\n",
        "  for layer in layer_dict: #replacing vanilla ReLU with custom Relu\n",
        "      if layer.activation == tf.keras.activations.relu:\n",
        "          layer.activation = backpropRelu\n",
        "\n",
        "  def normalize(img):\n",
        "      img = tf.clip_by_value(img, clip_value_min=0, clip_value_max=1)\n",
        "      #img = (img - tf.reduce_min(img)) / (tf.reduce_max(img) - tf.reduce_min(img))\n",
        "      return img.numpy()\n",
        "\n",
        "  def Plot_Image(img,nrow,ncol,index):\n",
        "    plt.subplot(nrow,ncol,index)\n",
        "    img_copy = np.zeros(shape=(224,224,3))\n",
        "    img_copy[img>0] = 1\n",
        "    img_copy[:,:,0] = img_copy.max(axis=2)\n",
        "    img_copy[:,:,1] = img_copy.max(axis=2)\n",
        "    img_copy[:,:,2] = img_copy.max(axis=2)\n",
        "\n",
        "    img = input_image[0]*img_copy\n",
        "    plt.imshow(img,interpolation='nearest')\n",
        "    plt.xticks([]);plt.yticks([])\n",
        "\n",
        "\n",
        "  def findMostActivated(tensor, top = 5): #finds top 5 most activated neurons from tensor size (14,14,512)\n",
        "      t = np.array(tensor)\n",
        "      t_flat = t.flatten()\n",
        "\n",
        "      def access_hwd(position): #returns 3D position of neuron from 1D index\n",
        "          height = (position//64)//54\n",
        "          width = (position//64)%54\n",
        "          depth = position - (64*54*height + 64*width)\n",
        "          return height, width, depth\n",
        "\n",
        "      ind = np.argpartition(t_flat, -top)[-top:] #gets indices of top 5 values from t_flat\n",
        "      ind = ind[np.argsort(t_flat[ind])][::-1]\n",
        "      return np.array([access_hwd(i) for i in ind])\n",
        "\n",
        "  def trackEffect(input_img, Top = 5):\n",
        "\n",
        "      with tf.GradientTape(persistent=True) as tape:\n",
        "          tape.watch(input_img) #watching input_img. After chain rule, gradient will eventually be against input_img.\n",
        "          conv = test_model(input_img)[0] #output of last convolutional layer\n",
        "          most_activated = findMostActivated(conv, top = Top) #top 5 activated neurons\n",
        "\n",
        "          plt.figure(figsize=(10,3))\n",
        "          plt.suptitle('Patch Vizualisation\\n'+title,fontweight=\"bold\",y=1+1e-2)\n",
        "          ind = 1\n",
        "          for i in range(Top):\n",
        "              x,y,z = most_activated[i]\n",
        "              max_conv = conv[x,y,z]\n",
        "              grads = tape.gradient(max_conv, input_img)\n",
        "              Plot_Image(img=normalize(grads[0]),nrow=1,ncol=5,index=ind)\n",
        "              ind+=1\n",
        "          plt.tight_layout()\n",
        "          plt.show()\n",
        "      return most_activated, conv\n",
        "  top5_neurons, output = trackEffect(input_image)\n",
        "\n",
        "Back_Prop(img_butterfly_colour,\"Butterfly\")\n",
        "Back_Prop(img_kangaroo_colour,\"Kangaroo\")\n",
        "Back_Prop(img_sunflower_colour,\"Sunflower\")"
      ],
      "metadata": {
        "id": "ZQV_TGpDJZRW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "d3N2pexnJaEj"
      },
      "source": [
        "# **`Q4`**\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "n7N0GEQbWw3X"
      },
      "outputs": [],
      "source": [
        "#VGG\n",
        "def vgg19_layer(x_train, y_train, x_val, y_val, x_test, y_test, batch_size,error_th,p,epochs=1e3):\n",
        "\n",
        "  base_model = VGG19(input_shape=(224,224,3), weights='imagenet', include_top=False)\n",
        "  base_model.trainable = False\n",
        "\n",
        "  num_classes=3\n",
        "\n",
        "  flat = Flatten()(base_model.layers[-1].output)\n",
        "  outputs = Dense(num_classes, activation='softmax')(flat)\n",
        "  model = keras.Model(base_model.inputs, outputs)\n",
        "\n",
        "  tensorflow.keras.optimizers.Adam(learning_rate=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08)\n",
        "  es = EarlyStopping(monitor='loss',mode='min',min_delta = error_th,verbose=1,patience=p)\n",
        "  model.compile(loss='categorical_crossentropy', optimizer='Adam', metrics=['accuracy'])\n",
        "\n",
        "  # this is only for training purpose\n",
        "  train_datagen = ImageDataGenerator(rescale = 1/255.0)\n",
        "  val_datagen   = ImageDataGenerator(rescale = 1/255.0)\n",
        "\n",
        "  x_train1 = train_datagen.flow_from_directory('/content/Group_21/train',target_size = (224, 224),\n",
        "                                               batch_size = batch_size,class_mode = 'categorical')\n",
        "\n",
        "  x_val1   = val_datagen.flow_from_directory('/content/Group_21/val',target_size = (224, 224),\n",
        "                                               batch_size = batch_size,class_mode = 'categorical')\n",
        "\n",
        "  history = model.fit(x_train1,validation_data=x_val1,epochs=int(epochs), batch_size=batch_size,callbacks=[es])\n",
        "\n",
        "  # this is for prediction purpose\n",
        "  y_train_pred = model.predict(x_train)\n",
        "  y_val_pred   = model.predict(x_val)\n",
        "  y_test_pred  = model.predict(x_test)\n",
        "\n",
        "  y_train_pred = ohetolabel(y_train_pred)\n",
        "  y_val_pred   = ohetolabel(y_val_pred)\n",
        "  y_test_pred  = ohetolabel(y_test_pred)\n",
        "\n",
        "\n",
        "  print(\"No of Epochs =\",len(history.history[\"loss\"]))\n",
        "  plt.figure(figsize=(12.5,8))\n",
        "  plt.suptitle('VGG19',fontweight=\"bold\",y=1+1e-2)\n",
        "  plt.subplot(1,2,2)\n",
        "  plt.plot(history.history['accuracy'],marker='.')\n",
        "  plt.plot(history.history['val_accuracy'],marker='.')\n",
        "  plt.title('Model accuracy')\n",
        "  plt.ylabel('Accuracy')\n",
        "  plt.xlabel('Epoch')\n",
        "  plt.legend(['Train', 'Validation'], loc='upper left')\n",
        "  plt.grid()\n",
        "\n",
        "  plt.subplot(1,2,1)\n",
        "  plt.plot(history.history['loss'],marker='.')\n",
        "  plt.plot(history.history['val_loss'],marker='.')\n",
        "  plt.title('Model loss')\n",
        "  plt.ylabel('Loss')\n",
        "  plt.xlabel('Epoch')\n",
        "  plt.legend(['Train', 'Validation'], loc='upper left')\n",
        "  plt.grid()\n",
        "  plt.tight_layout();plt.show()\n",
        "\n",
        "  print(\"Training Accuracy =\\t\",history.history['accuracy'][-1])\n",
        "  print(\"Validation Accuracy =\\t\",history.history['val_accuracy'][-1])\n",
        "\n",
        "  #Generate the confusion matrix\n",
        "  plt.figure(figsize=(12.5,4))\n",
        "  plt.suptitle('VGG19',fontweight=\"bold\",y=1+1e-2)\n",
        "  plt.subplot(1,3,1)\n",
        "  cf_matrix = confusion_matrix(y_train, y_train_pred)\n",
        "  cf_matrix_plot(cf_matrix,\"Training Data\")\n",
        "\n",
        "  plt.subplot(1,3,2)\n",
        "  cf_matrix = confusion_matrix(y_val, y_val_pred)\n",
        "  cf_matrix_plot(cf_matrix,\"Validation Data\")\n",
        "\n",
        "  plt.subplot(1,3,3)\n",
        "  cf_matrix = confusion_matrix(y_test, y_test_pred)\n",
        "  cf_matrix_plot(cf_matrix,\"Test Data\")\n",
        "  plt.tight_layout()\n",
        "  plt.show()\n",
        "\n",
        "  print(\"\\tTraining Data\\t\\t\\t\\tValidation Data\\t\\t\\t\\tTest Data\")\n",
        "  print(\"Accuracy:\",end=\" \")\n",
        "  print(np.around(accuracy_score(y_train , y_train_pred),4),end=\"\\t\"*4)\n",
        "  print(np.around(accuracy_score(y_val , y_val_pred),4),end=\"\\t\"*5)\n",
        "  print(np.around(accuracy_score(y_test , y_test_pred),4))\n",
        "\n",
        "  print(\"Precision:\",end=\"\")\n",
        "  print(np.around(precision_score(y_train , y_train_pred,average=\"macro\"),4),end=\"\\t\"*4)\n",
        "  print(np.around(precision_score(y_val , y_val_pred,average=\"macro\"),4),end=\"\\t\"*5)\n",
        "  print(np.around(precision_score(y_test , y_test_pred,average=\"macro\"),4))\n",
        "\n",
        "  print(\"Recall:\",end=\"   \")\n",
        "  print(np.around(recall_score(y_train , y_train_pred,average=\"macro\"),4),end=\"\\t\"*4)\n",
        "  print(np.around(recall_score(y_val , y_val_pred,average=\"macro\"),4),end=\"\\t\"*5)\n",
        "  print(np.around(recall_score(y_test , y_test_pred,average=\"macro\"),4))\n",
        "\n",
        "  print(\"F1-Score:\",end=\" \")\n",
        "  print(np.around(f1_score(y_train , y_train_pred,average=\"macro\"),4),end=\"\\t\"*4)\n",
        "  print(np.around(f1_score(y_val , y_val_pred,average=\"macro\"),4),end=\"\\t\"*5)\n",
        "  print(np.around(f1_score(y_test , y_test_pred,average=\"macro\"),4))\n",
        "\n",
        "  return base_model,model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0AlimO2EbmeR"
      },
      "outputs": [],
      "source": [
        "error_th = 1e-2\n",
        "p = 1\n",
        "batch_size = 10\n",
        "\n",
        "base_model,VGG_model = vgg19_layer(x_train, y_train, x_val, y_val, x_test, y_test, batch_size,error_th,p,epochs=100)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def Back_Prop(input_image,title):\n",
        "  input_image = tf.convert_to_tensor(input_image.reshape(-1,224,224,3))\n",
        "  base_model = VGG19(weights='imagenet')\n",
        "  test_model = Model(inputs=base_model.inputs, outputs=base_model.get_layer('block5_pool').output)\n",
        "\n",
        "  result = base_model(input_image)\n",
        "\n",
        "  @tf.custom_gradient\n",
        "  def backpropRelu(x):\n",
        "      def grad(dy):\n",
        "          return tf.cast(dy>0,tf.float32)  * tf.cast(x>0,tf.float32) * dy\n",
        "      return tf.nn.relu(x), grad\n",
        "\n",
        "  layer_dict = [layer for layer in test_model.layers[1:] if hasattr(layer,'activation')]\n",
        "  for layer in layer_dict: #replacing vanilla ReLU with custom Relu\n",
        "      if layer.activation == tf.keras.activations.relu:\n",
        "          layer.activation = backpropRelu\n",
        "\n",
        "  def normalize(img):\n",
        "      img = tf.clip_by_value(img, clip_value_min=0, clip_value_max=1)\n",
        "      #img = (img - tf.reduce_min(img)) / (tf.reduce_max(img) - tf.reduce_min(img))\n",
        "      return img.numpy()\n",
        "\n",
        "  def Plot_Image(img,nrow,ncol,index):\n",
        "    plt.subplot(nrow,ncol,index)\n",
        "    img_copy = np.zeros(shape=(224,224,3))\n",
        "    img_copy[img>0] = 1\n",
        "    img_copy[:,:,0] = img_copy.max(axis=2)\n",
        "    img_copy[:,:,1] = img_copy.max(axis=2)\n",
        "    img_copy[:,:,2] = img_copy.max(axis=2)\n",
        "\n",
        "    img = input_image[0]*img_copy\n",
        "    plt.imshow(img,interpolation='nearest')\n",
        "    plt.xticks([]);plt.yticks([])\n",
        "\n",
        "  def findMostActivated(tensor, top = 5): #finds top 5 most activated neurons from tensor size (14,14,512)\n",
        "      t = np.array(tensor)\n",
        "      t_flat = t.flatten()\n",
        "\n",
        "      def access_hwd(position): #returns 3D position of neuron from 1D index\n",
        "          height = (position//512)//7\n",
        "          width = (position//512)%7\n",
        "          depth = position - (512*7*height + 512*width)\n",
        "          return height, width, depth\n",
        "\n",
        "      ind = np.argpartition(t_flat, -top)[-top:] #gets indices of top 5 values from t_flat\n",
        "      ind = ind[np.argsort(t_flat[ind])][::-1]\n",
        "      return np.array([access_hwd(i) for i in ind])\n",
        "\n",
        "  def trackEffect(input_img, Top = 5):\n",
        "\n",
        "      with tf.GradientTape(persistent=True) as tape:\n",
        "          tape.watch(input_img) #watching input_img. After chain rule, gradient will eventually be against input_img.\n",
        "          conv = test_model(input_img)[0] #output of last convolutional layer\n",
        "          most_activated = findMostActivated(conv, top = Top) #top 5 activated neurons\n",
        "\n",
        "          plt.figure(figsize=(10,3))\n",
        "          plt.suptitle('Patch Vizualisation\\n'+title,fontweight=\"bold\",y=1+1e-2)\n",
        "          ind = 1\n",
        "          for i in range(Top):\n",
        "              x,y,z = most_activated[i]\n",
        "              max_conv = conv[x,y,z]\n",
        "              grads = tape.gradient(max_conv, input_img)\n",
        "              Plot_Image(img=normalize(grads[0]),nrow=1,ncol=5,index=ind)\n",
        "              ind+=1\n",
        "          plt.tight_layout()\n",
        "          plt.show()\n",
        "      return most_activated, conv\n",
        "  top5_neurons, output = trackEffect(input_image)\n",
        "\n",
        "Back_Prop(img_butterfly_colour,\"Butterfly\")\n",
        "Back_Prop(img_kangaroo_colour,\"Kangaroo\")\n",
        "Back_Prop(img_sunflower_colour,\"Sunflower\")"
      ],
      "metadata": {
        "id": "6hau-8tJJP1e"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def Back_Prop(input_image,title):\n",
        "  input_image = tf.convert_to_tensor(input_image.reshape(-1,224,224,3))\n",
        "  base_model = VGG19(weights='imagenet')\n",
        "  test_model = Model(inputs=base_model.inputs, outputs=base_model.get_layer('block5_conv4').output)\n",
        "\n",
        "  result = base_model(input_image)\n",
        "  #print(decode_predictions(result.numpy()))\n",
        "\n",
        "  @tf.custom_gradient\n",
        "  def backpropRelu(x):\n",
        "      def grad(dy):\n",
        "          return tf.cast(dy>0,tf.float32)  * tf.cast(x>0,tf.float32) * dy\n",
        "      return tf.nn.relu(x), grad\n",
        "\n",
        "  layer_dict = [layer for layer in test_model.layers[1:] if hasattr(layer,'activation')]\n",
        "  for layer in layer_dict: #replacing vanilla ReLU with custom Relu\n",
        "      if layer.activation == tf.keras.activations.relu:\n",
        "          layer.activation = backpropRelu\n",
        "\n",
        "  def normalize(img):\n",
        "      img = (img - tf.reduce_min(img)) / (tf.reduce_max(img) - tf.reduce_min(img))\n",
        "      return img\n",
        "\n",
        "  def Plot_Image(img,nrow,ncol,index):\n",
        "    plt.subplot(nrow,ncol,index)\n",
        "    plt.imshow(img)\n",
        "    plt.xticks([]);plt.yticks([])\n",
        "\n",
        "  def findMostActivated(tensor, top = 5): #finds top 5 most activated neurons from tensor size (14,14,512)\n",
        "      t = np.array(tensor)\n",
        "      t_flat = t.flatten()\n",
        "\n",
        "      def access_hwd(position): #returns 3D position of neuron from 1D index\n",
        "          height = (position//512)//14\n",
        "          width = (position//512)%14\n",
        "          depth = position - (512*14*height + 512*width)\n",
        "          return height, width, depth\n",
        "\n",
        "      ind = np.argpartition(t_flat, -top)[-top:] #gets indices of top 5 values from t_flat\n",
        "      ind = ind[np.argsort(t_flat[ind])][::-1]\n",
        "      return np.array([access_hwd(i) for i in ind])\n",
        "\n",
        "  def trackEffect(input_img, Top = 5):\n",
        "\n",
        "      with tf.GradientTape(persistent=True) as tape:\n",
        "          tape.watch(input_img) #watching input_img. After chain rule, gradient will eventually be against input_img.\n",
        "          conv = test_model(input_img)[0] #output of last convolutional layer\n",
        "          most_activated = findMostActivated(conv, top = Top) #top 5 activated neurons\n",
        "\n",
        "          plt.figure(figsize=(10,3))\n",
        "          plt.suptitle('Guided-Backpropagation\\n'+title,fontweight=\"bold\",y=1+1e-2)\n",
        "          ind = 1\n",
        "          for i in range(Top):\n",
        "              x,y,z = most_activated[i]\n",
        "              max_conv = conv[x,y,z]\n",
        "              grads = tape.gradient(max_conv, input_img)\n",
        "              Plot_Image(img=normalize(grads[0]),nrow=1,ncol=5,index=ind)\n",
        "              ind+=1\n",
        "          plt.tight_layout()\n",
        "          plt.show()\n",
        "      return most_activated, conv\n",
        "  top5_neurons, output = trackEffect(input_image)\n",
        "\n",
        "Back_Prop(img_butterfly_colour,\"Butterfly\")\n",
        "Back_Prop(img_kangaroo_colour,\"Kangaroo\")\n",
        "Back_Prop(img_sunflower_colour,\"Sunflower\")"
      ],
      "metadata": {
        "id": "Pnigt5LkNIZ_"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}