nn_2b_assignment_v1_2.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/thinkGrow/Neural-Network-CSC424/blob/main/nn_2b_assignment_v1_2.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102,
          "referenced_widgets": [
            "bda97dab5fa0491b8421aee2fad29f17",
            "c49125a25f30451ea169a899ec9bd78a",
            "1ccf72e6f77f482cbc338e579520f06e",
            "7efcfd87f1974da09d9063ebfb6c2d6e",
            "2194a4b7945e4bae939cde9269f33ad6",
            "5d8dc1bbfa5e43f28b830fd72fba4cc3",
            "5d391a699c004cb9bb2ed1206cf2e683",
            "d6e41cee6181471281f39c8f719b33a6",
            "3e91c55c4c7f4760a648b75cae70e9a8",
            "d7d8502bd2864e61ad7693294685eff2",
            "f0465056deb9447ca798b8aac5066776"
          ]
        },
        "id": "-C8buUG17npB",
        "outputId": "fcc7fc04-c34b-46fa-af3a-7e73676fc393"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ./data/cifar-10-python.tar.gz\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "bda97dab5fa0491b8421aee2fad29f17",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/170498071 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Extracting ./data/cifar-10-python.tar.gz to ./data\n",
            "Files already downloaded and verified\n"
          ]
        }
      ],
      "source": [
        "import torch\n",
        "import torchvision\n",
        "import torchvision.transforms as transforms\n",
        "import numpy as np\n",
        "import math\n",
        "\n",
        "trainset = torchvision.datasets.CIFAR10(root='./data', train=True, download=True)\n",
        "testset = torchvision.datasets.CIFAR10(root='./data', train=False, download=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lzV2A5BPbSXv"
      },
      "outputs": [],
      "source": [
        "x_train = trainset.data\n",
        "x_test = testset.data\n",
        "y_train = np.array(trainset.targets)\n",
        "y_test = np.array(testset.targets)\n",
        "x_train = x_train/255\n",
        "x_test = x_test/255"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "N0ZhMW4vbZtp"
      },
      "outputs": [],
      "source": [
        "x_train = x_train[np.any([y_train==3,y_train==5], axis=0)] #store all x_train with class 3 and 5\n",
        "y_train = y_train[np.any([y_train==3,y_train==5], axis=0)] #store all y_train with class 3 and 5\n",
        "x_test =  x_test[np.any([y_test==3,y_test==5], axis=0)]\n",
        "y_test = y_test[np.any([y_test==3,y_test==5], axis=0)]\n",
        "\n",
        "y_train = np.where(y_train==3, 0, 1) #class 3 becomes 0 and class 1 becomes 5.\n",
        "y_test = np.where(y_test==3, 0, 1)\n",
        "\n",
        "# class 0 = cat\n",
        "# class 1 = dog\n",
        "\n",
        "#checked and verified"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CxiEd1AkbbT1"
      },
      "outputs": [],
      "source": [
        "x_train = x_train.reshape(x_train.shape[0], 3072) #10,000 x 3072\n",
        "x_test = x_test.reshape(x_test.shape[0], 3072) #2000 x 3072\n",
        "#print(x_test_rows.shape)\n",
        "y_train = y_train.reshape(10000,1)\n",
        "y_test = y_test.reshape(2000,1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tltZ22lr7x97"
      },
      "outputs": [],
      "source": [
        "def sigmoid( signal ):\n",
        "    # Prevent overflow.\n",
        "    signal = np.clip( signal, -500, 500 )\n",
        "    # Calculate activation signal\n",
        "    signal = 1.0/( 1 + np.exp( -signal ))\n",
        "    return signal\n",
        "\n",
        "\n",
        "def sigmoid_der(x):\n",
        "    return x *(1-x)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true,
          "base_uri": "https://localhost:8080/"
        },
        "id": "p7i6-H6B70Ai",
        "outputId": "0598cdac-fdcb-444a-ebd6-de6bb23ceb4a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch no. 0 : 1253.319967016273\n",
            "Epoch no. 1 : 1251.9909640512897\n",
            "Epoch no. 2 : 1250.9081535425476\n",
            "Epoch no. 3 : 1249.8543310248847\n",
            "Epoch no. 4 : 1248.8283142427688\n",
            "Epoch no. 5 : 1247.829206563545\n",
            "Epoch no. 6 : 1246.8561389224628\n",
            "Epoch no. 7 : 1245.9082683753143\n",
            "Epoch no. 8 : 1244.9847777406944\n",
            "Epoch no. 9 : 1244.084875203409\n"
          ]
        }
      ],
      "source": [
        "class NeuralNetwork:\n",
        "    def __init__(self):\n",
        "        #random.rand = random values given in a shape.\n",
        "\n",
        "        fan_in_h = 3072\n",
        "        fan_out_h = 4000\n",
        "        limit_h = math.sqrt(6/(fan_in_h + fan_out_h))\n",
        "\n",
        "        fan_in_o = 4000\n",
        "        fan_out_o = 1\n",
        "        limit_o = math.sqrt(6/(fan_in_o + fan_out_o))\n",
        "\n",
        "        self.wh = np.random.uniform(-limit_h, limit_h, size=(fan_in_h, fan_out_h)) #w1 to w(3,072*4,000) : rand(input,hidden)\n",
        "        self.wo = np.random.uniform(-limit_o, limit_o, size=(fan_in_o, fan_out_o))\n",
        "        self.lr = 0.000001  \n",
        "        self.batch_size = 1000\n",
        "\n",
        "\n",
        "    def train(self, x_train, y_train):\n",
        "        num_of_iteration = 0 \n",
        "        mul_of_iteration = 1\n",
        "        \n",
        "        for epoch in range(10):\n",
        "            # x_train.shape = 10,000*3,072\n",
        "            i = 0\n",
        "            j = self.batch_size\n",
        "\n",
        "            for i in range(10000//self.batch_size):\n",
        "                x_batch = x_train[i:j,:]\n",
        "                y_batch = y_train[i:j,:]\n",
        "                i += self.batch_size\n",
        "                j += self.batch_size\n",
        "\n",
        "                #feed forward\n",
        "                num_of_iteration += 1\n",
        "                zh = np.dot(x_batch, self.wh)\n",
        "                ah = sigmoid(zh) #10,000*4,000\n",
        "                zo = np.dot(ah, self.wo) #10,000*1\n",
        "                ao = sigmoid(zo) #10,000*1\n",
        "                \n",
        "                error_out = ( 0.5 * (np.power((ao - y_batch), 2))) \n",
        "                #prints out error after every mul_of_iteration-th iteration\n",
        "                \n",
        "\n",
        "                #back propagate\n",
        "\n",
        "                #local gradients\n",
        "                dcost_dao = ao - y_batch #10,000*1\n",
        "                dzo_dwo = ah\n",
        "                dao_dzo = sigmoid_der(ao) \n",
        "                dzo_dah = self.wo\n",
        "                dzh_dwh = x_batch\n",
        "                dah_dzh = sigmoid_der(ah)\n",
        "\n",
        "                #upstream gradients = downstream gradients * local gradient\n",
        "                dcost_dzo = dcost_dao * dao_dzo\n",
        "                dcost_dwo = np.dot(dzo_dwo.T, dcost_dzo)\n",
        "                dcost_dah = np.dot(dcost_dzo , dzo_dah.T)\n",
        "                dcost_dzh = dah_dzh * dcost_dah\n",
        "                dcost_dwh = np.dot(dzh_dwh.T, dcost_dzh)\n",
        "\n",
        "                # Update Weights ================\n",
        "                self.wh -= self.lr * dcost_dwh\n",
        "                self.wo -= self.lr * dcost_dwo\n",
        "            print( f'Epoch no. {epoch} : {error_out.sum()}' )\n",
        "            \n",
        "            \n",
        "            \n",
        "    def predict(self, x_test):\n",
        "        # feedforward\n",
        "        zh = np.dot(x_test, self.wh)\n",
        "        ah = sigmoid(zh)\n",
        "\n",
        "        zo = np.dot(ah, self.wo)\n",
        "        ao = sigmoid(zo)            \n",
        "        return ao\n",
        "        \n",
        "\n",
        "nn = NeuralNetwork()\n",
        "nn.train(x_train,y_train)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "CKzuVTDG73Xx"
      },
      "outputs": [],
      "source": [
        "#test\n",
        "y_predict = nn.predict(x_test)\n",
        "\n",
        "#binary classification\n",
        "y_predict = np.where(y_predict<0.5, 0, 1 )\n",
        "'''what np.where does : \n",
        "    if score < 0.5 : label 0  \n",
        "    else : label 1  ''';"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "iOBpMQ-gvhJ4",
        "outputId": "2f790881-4a97-461d-f359-f91cae628c0c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " Test Accuracy : 0.5445\n"
          ]
        }
      ],
      "source": [
        "#Accuracy\n",
        "import pandas as pd\n",
        "from sklearn.metrics import accuracy_score\n",
        "\n",
        "print(f' Test Accuracy : { accuracy_score(y_test, y_predict) }')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "zhRGL9JTviah"
      },
      "outputs": [],
      "source": [
        ""
      ]
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [],
      "name": "nn_2b_assignment_v1.2.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyOWBLwHP3M6+riQdyNvfKaH",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "1ccf72e6f77f482cbc338e579520f06e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d6e41cee6181471281f39c8f719b33a6",
            "max": 170498071,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_3e91c55c4c7f4760a648b75cae70e9a8",
            "value": 170498071
          }
        },
        "2194a4b7945e4bae939cde9269f33ad6": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3e91c55c4c7f4760a648b75cae70e9a8": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "5d391a699c004cb9bb2ed1206cf2e683": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5d8dc1bbfa5e43f28b830fd72fba4cc3": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7efcfd87f1974da09d9063ebfb6c2d6e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d7d8502bd2864e61ad7693294685eff2",
            "placeholder": "​",
            "style": "IPY_MODEL_f0465056deb9447ca798b8aac5066776",
            "value": " 170499072/? [00:11&lt;00:00, 17499341.05it/s]"
          }
        },
        "bda97dab5fa0491b8421aee2fad29f17": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_c49125a25f30451ea169a899ec9bd78a",
              "IPY_MODEL_1ccf72e6f77f482cbc338e579520f06e",
              "IPY_MODEL_7efcfd87f1974da09d9063ebfb6c2d6e"
            ],
            "layout": "IPY_MODEL_2194a4b7945e4bae939cde9269f33ad6"
          }
        },
        "c49125a25f30451ea169a899ec9bd78a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5d8dc1bbfa5e43f28b830fd72fba4cc3",
            "placeholder": "​",
            "style": "IPY_MODEL_5d391a699c004cb9bb2ed1206cf2e683",
            "value": ""
          }
        },
        "d6e41cee6181471281f39c8f719b33a6": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d7d8502bd2864e61ad7693294685eff2": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f0465056deb9447ca798b8aac5066776": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}