simple_neural_network.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Matin Ghorbani 6.8.Assignment\n",
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Implement a simple neural network (Multi Layer Perceptron) on Scikit-Learn digits dataset\n",
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import Dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.datasets import load_digits\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1797, 64)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset = load_digits()\n",
    "dataset.data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZgAAAGdCAYAAAAv9mXmAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAX+0lEQVR4nO3df2zUhf3H8dfRWw8m7QlIoR39gYoiYDukQFhxgiCkQQL7gxGCWYFtieQYYGNi+s9Ksoxjf2xBF1KBsULiOnDLWp0ZdMBsySIdpaQJaIKgIJ0InQtcf/xxmN7n+9du6xcp/Rx998PneD6ST+JdPse9QpAnn7u2F3AcxxEAAENshNcDAADpicAAAEwQGACACQIDADBBYAAAJggMAMAEgQEAmCAwAAATweF+wkQioatXryorK0uBQGC4nx4AcA8cx1F3d7fy8vI0YsTA1yjDHpirV68qPz9/uJ8WADCEOjo6NGnSpAHPGfbAZGVlDfdTwscKCgq8npCSuro6ryekZP78+V5PgE8M5u/yYQ8ML4vBjbtdgt+vRo8e7fUEwNRg/i735/+9AID7HoEBAJggMAAAEwQGAGCCwAAATBAYAIAJAgMAMEFgAAAmCAwAwASBAQCYIDAAABMEBgBggsAAAEwQGACACQIDADBBYAAAJlIKzK5du1RUVKSRI0dq7ty5OnXq1FDvAgD4nOvAHDp0SJWVlaqurtaZM2dUUlKipUuXqrOz02IfAMCnXAfmV7/6lX784x9r/fr1mjZtmt58801985vf1G9/+1uLfQAAn3IVmFu3bqmtrU2LFy/+7y8wYoQWL16skydPfu1j4vG4urq6+h0AgPTnKjBffvml+vr6NGHChH73T5gwQdeuXfvax0SjUYXD4eSRn5+f+loAgG+YfxVZVVWVYrFY8ujo6LB+SgDAfSDo5uRHHnlEGRkZun79er/7r1+/rokTJ37tY0KhkEKhUOoLAQC+5OoKJjMzU7NmzdLx48eT9yUSCR0/flzz5s0b8nEAAP9ydQUjSZWVlaqoqFBpaanmzJmjnTt3qre3V+vXr7fYBwDwKdeBWb16tf71r3/ppz/9qa5du6Zvf/vbOnLkyG1v/AMAHmyuAyNJmzZt0qZNm4Z6CwAgjfCzyAAAJggMAMAEgQEAmCAwAAATBAYAYILAAABMEBgAgAkCAwAwQWAAACYIDADABIEBAJggMAAAEwQGAGCCwAAATBAYAICJlD4PBhgu69at83pCSoqKiryeAHiOKxgAgAkCAwAwQWAAACYIDADABIEBAJggMAAAEwQGAGCCwAAATBAYAIAJAgMAMEFgAAAmCAwAwASBAQCYIDAAABMEBgBggsAAAEwQGACACQIDADBBYAAAJlwH5sSJE1q+fLny8vIUCATU0NBgMAsA4HeuA9Pb26uSkhLt2rXLYg8AIE0E3T6gvLxc5eXlFlsAAGnEdWDcisfjisfjydtdXV3WTwkAuA+Yv8kfjUYVDoeTR35+vvVTAgDuA+aBqaqqUiwWSx4dHR3WTwkAuA+Yv0QWCoUUCoWsnwYAcJ/h+2AAACZcX8H09PTo4sWLyduXLl1Se3u7xo4dq4KCgiEdBwDwL9eBOX36tBYuXJi8XVlZKUmqqKjQ/v37h2wYAMDfXAdmwYIFchzHYgsAII3wHgwAwASBAQCYIDAAABMEBgBggsAAAEwQGACACQIDADBBYAAAJggMAMAEgQEAmCAwAAATBAYAYILAAABMEBgAgAkCAwAw4frzYOA/K1as8HpCyqqrq72ekJKVK1d6PSElRUVFXk9IyeXLl72egK/BFQwAwASBAQCYIDAAABMEBgBggsAAAEwQGACACQIDADBBYAAAJggMAMAEgQEAmCAwAAATBAYAYILAAABMEBgAgAkCAwAwQWAAACYIDADABIEBAJggMAAAE64CE41GNXv2bGVlZSknJ0crV67U+fPnrbYBAHzMVWCam5sViUTU0tKio0eP6quvvtKSJUvU29trtQ8A4FNBNycfOXKk3+39+/crJydHbW1t+u53vzukwwAA/uYqMP9fLBaTJI0dO/aO58TjccXj8eTtrq6ue3lKAIBPpPwmfyKR0NatW1VWVqYZM2bc8bxoNKpwOJw88vPzU31KAICPpByYSCSic+fO6eDBgwOeV1VVpVgsljw6OjpSfUoAgI+k9BLZpk2b9N577+nEiROaNGnSgOeGQiGFQqGUxgEA/MtVYBzH0U9+8hPV19erqalJkydPttoFAPA5V4GJRCKqq6vTO++8o6ysLF27dk2SFA6HNWrUKJOBAAB/cvUeTE1NjWKxmBYsWKDc3NzkcejQIat9AACfcv0SGQAAg8HPIgMAmCAwAAATBAYAYILAAABMEBgAgAkCAwAwQWAAACYIDADABIEBAJggMAAAEwQGAGCCwAAATBAYAIAJAgMAMEFgAAAmCAwAwETAGeZPEevq6lI4HB7Op3zg3bx50+sJKWtvb/d6QkpWrlzp9YSU3Lhxw+sJKVm4cKHXE1LW1NTk9YSUxGIxZWdnD3gOVzAAABMEBgBggsAAAEwQGACACQIDADBBYAAAJggMAMAEgQEAmCAwAAATBAYAYILAAABMEBgAgAkCAwAwQWAAACYIDADABIEBAJggMAAAEwQGAGDCVWBqampUXFys7OxsZWdna968eTp8+LDVNgCAj7kKzKRJk7Rjxw61tbXp9OnTev7557VixQp9+OGHVvsAAD4VdHPy8uXL+93++c9/rpqaGrW0tGj69OlDOgwA4G+uAvO/+vr69Ic//EG9vb2aN2/eHc+Lx+OKx+PJ211dXak+JQDAR1y/yX/27FmNHj1aoVBIL7/8surr6zVt2rQ7nh+NRhUOh5NHfn7+PQ0GAPiD68A8+eSTam9v1z/+8Q9t3LhRFRUV+uijj+54flVVlWKxWPLo6Oi4p8EAAH9w/RJZZmamHn/8cUnSrFmz1Nraqtdff127d+/+2vNDoZBCodC9rQQA+M49fx9MIpHo9x4LAACSyyuYqqoqlZeXq6CgQN3d3aqrq1NTU5MaGxut9gEAfMpVYDo7O/WDH/xAX3zxhcLhsIqLi9XY2KgXXnjBah8AwKdcBWbfvn1WOwAAaYafRQYAMEFgAAAmCAwAwASBAQCYIDAAABMEBgBggsAAAEwQGACACQIDADBBYAAAJggMAMAEgQEAmCAwAAATBAYAYILAAABMEBgAgAlXHzj2oFuwYIHXE1ISDoe9npCydevWeT0hJdu2bfN6wgPFr/9vSlJTU5PXE8xwBQMAMEFgAAAmCAwAwASBAQCYIDAAABMEBgBggsAAAEwQGACACQIDADBBYAAAJggMAMAEgQEAmCAwAAATBAYAYILAAABMEBgAgAkCAwAwQWAAACbuKTA7duxQIBDQ1q1bh2gOACBdpByY1tZW7d69W8XFxUO5BwCQJlIKTE9Pj9auXau9e/dqzJgxQ70JAJAGUgpMJBLRsmXLtHjx4qHeAwBIE0G3Dzh48KDOnDmj1tbWQZ0fj8cVj8eTt7u6utw+JQDAh1xdwXR0dGjLli363e9+p5EjRw7qMdFoVOFwOHnk5+enNBQA4C+uAtPW1qbOzk4988wzCgaDCgaDam5u1htvvKFgMKi+vr7bHlNVVaVYLJY8Ojo6hmw8AOD+5eolskWLFuns2bP97lu/fr2mTp2q1157TRkZGbc9JhQKKRQK3dtKAIDvuApMVlaWZsyY0e++hx56SOPGjbvtfgDAg43v5AcAmHD9VWT/X1NT0xDMAACkG65gAAAmCAwAwASBAQCYIDAAABMEBgBggsAAAEwQGACACQIDADBBYAAAJggMAMAEgQEAmCAwAAATBAYAYILAAABMEBgAgAkCAwAwcc8fOPYg8euHqx04cMDrCSnz6+95YWGh1xMeKH79c5LuuIIBAJggMAAAEwQGAGCCwAAATBAYAIAJAgMAMEFgAAAmCAwAwASBAQCYIDAAABMEBgBggsAAAEwQGACACQIDADBBYAAAJggMAMAEgQEAmCAwAAATBAYAYMJVYLZt26ZAINDvmDp1qtU2AICPBd0+YPr06Tp27Nh/f4Gg618CAPAAcF2HYDCoiRMnWmwBAKQR1+/BXLhwQXl5eXr00Ue1du1aXblyZcDz4/G4urq6+h0AgPTnKjBz587V/v37deTIEdXU1OjSpUt69tln1d3dfcfHRKNRhcPh5JGfn3/PowEA9z9XgSkvL9eqVatUXFyspUuX6i9/+Ytu3rypt99++46PqaqqUiwWSx4dHR33PBoAcP+7p3foH374YT3xxBO6ePHiHc8JhUIKhUL38jQAAB+6p++D6enp0SeffKLc3Nyh2gMASBOuAvPqq6+qublZly9f1gcffKDvfe97ysjI0Jo1a6z2AQB8ytVLZP/85z+1Zs0a/fvf/9b48eM1f/58tbS0aPz48Vb7AAA+5SowBw8etNoBAEgz/CwyAIAJAgMAMEFgAAAmCAwAwASBAQCYIDAAABMEBgBggsAAAEwQGACACQIDADBBYAAAJggMAMAEgQEAmCAwAAATBAYAYMLV58HAn9atW+f1hAdOe3u71xNS0tDQ4PWElDQ1NXk9AV+DKxgAgAkCAwAwQWAAACYIDADABIEBAJggMAAAEwQGAGCCwAAATBAYAIAJAgMAMEFgAAAmCAwAwASBAQCYIDAAABMEBgBggsAAAEwQGACACQIDADBBYAAAJlwH5vPPP9dLL72kcePGadSoUXr66ad1+vRpi20AAB8Lujn5xo0bKisr08KFC3X48GGNHz9eFy5c0JgxY6z2AQB8ylVgfvGLXyg/P1+1tbXJ+yZPnjzkowAA/ufqJbJ3331XpaWlWrVqlXJycjRz5kzt3bt3wMfE43F1dXX1OwAA6c9VYD799FPV1NRoypQpamxs1MaNG7V582YdOHDgjo+JRqMKh8PJIz8//55HAwDufwHHcZzBnpyZmanS0lJ98MEHyfs2b96s1tZWnTx58msfE4/HFY/Hk7e7urqIDNJee3u71xNS0tDQ4PWElGzbts3rCQ+cWCym7OzsAc9xdQWTm5uradOm9bvvqaee0pUrV+74mFAopOzs7H4HACD9uQpMWVmZzp8/3+++jz/+WIWFhUM6CgDgf64C88orr6ilpUXbt2/XxYsXVVdXpz179igSiVjtAwD4lKvAzJ49W/X19fr973+vGTNm6Gc/+5l27typtWvXWu0DAPiUq++DkaQXX3xRL774osUWAEAa4WeRAQBMEBgAgAkCAwAwQWAAACYIDADABIEBAJggMAAAEwQGAGCCwAAATBAYAIAJAgMAMEFgAAAmCAwAwASBAQCYIDAAABMEBgBgwvUHjgFIX5cvX/Z6AtIIVzAAABMEBgBggsAAAEwQGACACQIDADBBYAAAJggMAMAEgQEAmCAwAAATBAYAYILAAABMEBgAgAkCAwAwQWAAACYIDADABIEBAJggMAAAEwQGAGDCVWCKiooUCARuOyKRiNU+AIBPBd2c3Nraqr6+vuTtc+fO6YUXXtCqVauGfBgAwN9cBWb8+PH9bu/YsUOPPfaYnnvuuSEdBQDwP1eB+V+3bt3SW2+9pcrKSgUCgTueF4/HFY/Hk7e7urpSfUoAgI+k/CZ/Q0ODbt68qXXr1g14XjQaVTgcTh75+fmpPiUAwEdSDsy+fftUXl6uvLy8Ac+rqqpSLBZLHh0dHak+JQDAR1J6ieyzzz7TsWPH9Kc//emu54ZCIYVCoVSeBgDgYyldwdTW1ionJ0fLli0b6j0AgDThOjCJREK1tbWqqKhQMJjy1wgAANKc68AcO3ZMV65c0YYNGyz2AADShOtLkCVLlshxHIstAIA0ws8iAwCYIDAAABMEBgBggsAAAEwQGACACQIDADBBYAAAJggMAMAEgQEAmCAwAAATBAYAYILAAABMEBgAgAkCAwAwQWAAACaG/SMp+SwZPAh6enq8npCSW7dueT0BPjGYv8uHPTDd3d3D/ZTAsJs/f77XEwBT3d3dCofDA54TcIb5kiKRSOjq1avKyspSIBAY0l+7q6tL+fn56ujoUHZ29pD+2pbYPbzYPfz8up3dt3McR93d3crLy9OIEQO/yzLsVzAjRozQpEmTTJ8jOzvbV38Y/oPdw4vdw8+v29nd392uXP6DN/kBACYIDADARFoFJhQKqbq6WqFQyOsprrB7eLF7+Pl1O7vvzbC/yQ8AeDCk1RUMAOD+QWAAACYIDADABIEBAJhIm8Ds2rVLRUVFGjlypObOnatTp055PemuTpw4oeXLlysvL0+BQEANDQ1eTxqUaDSq2bNnKysrSzk5OVq5cqXOnz/v9ay7qqmpUXFxcfKbz+bNm6fDhw97Pcu1HTt2KBAIaOvWrV5PGdC2bdsUCAT6HVOnTvV61qB8/vnneumllzRu3DiNGjVKTz/9tE6fPu31rLsqKiq67fc8EAgoEol4sictAnPo0CFVVlaqurpaZ86cUUlJiZYuXarOzk6vpw2ot7dXJSUl2rVrl9dTXGlublYkElFLS4uOHj2qr776SkuWLFFvb6/X0wY0adIk7dixQ21tbTp9+rSef/55rVixQh9++KHX0wattbVVu3fvVnFxsddTBmX69On64osvksff//53ryfd1Y0bN1RWVqZvfOMbOnz4sD766CP98pe/1JgxY7yedletra39fr+PHj0qSVq1apU3g5w0MGfOHCcSiSRv9/X1OXl5eU40GvVwlTuSnPr6eq9npKSzs9OR5DQ3N3s9xbUxY8Y4v/nNb7yeMSjd3d3OlClTnKNHjzrPPfecs2XLFq8nDai6utopKSnxeoZrr732mjN//nyvZwyJLVu2OI899piTSCQ8eX7fX8HcunVLbW1tWrx4cfK+ESNGaPHixTp58qSHyx4csVhMkjR27FiPlwxeX1+fDh48qN7eXs2bN8/rOYMSiUS0bNmyfn/W73cXLlxQXl6eHn30Ua1du1ZXrlzxetJdvfvuuyotLdWqVauUk5OjmTNnau/evV7Pcu3WrVt66623tGHDhiH/wcKD5fvAfPnll+rr69OECRP63T9hwgRdu3bNo1UPjkQioa1bt6qsrEwzZszwes5dnT17VqNHj1YoFNLLL7+s+vp6TZs2zetZd3Xw4EGdOXNG0WjU6ymDNnfuXO3fv19HjhxRTU2NLl26pGefffa+/8iOTz/9VDU1NZoyZYoaGxu1ceNGbd68WQcOHPB6misNDQ26efOm1q1b59mGYf9pykgvkUhE586d88Vr65L05JNPqr29XbFYTH/84x9VUVGh5ubm+zoyHR0d2rJli44ePaqRI0d6PWfQysvLk/9dXFysuXPnqrCwUG+//bZ++MMferhsYIlEQqWlpdq+fbskaebMmTp37pzefPNNVVRUeLxu8Pbt26fy8nLl5eV5tsH3VzCPPPKIMjIydP369X73X79+XRMnTvRo1YNh06ZNeu+99/T++++bfwTDUMnMzNTjjz+uWbNmKRqNqqSkRK+//rrXswbU1tamzs5OPfPMMwoGgwoGg2pubtYbb7yhYDCovr4+rycOysMPP6wnnnhCFy9e9HrKgHJzc2/7B8dTTz3li5f3/uOzzz7TsWPH9KMf/cjTHb4PTGZmpmbNmqXjx48n70skEjp+/LhvXlv3G8dxtGnTJtXX1+tvf/ubJk+e7PWklCUSCcXjca9nDGjRokU6e/as2tvbk0dpaanWrl2r9vZ2ZWRkeD1xUHp6evTJJ58oNzfX6ykDKisru+3L7j/++GMVFhZ6tMi92tpa5eTkaNmyZZ7uSIuXyCorK1VRUaHS0lLNmTNHO3fuVG9vr9avX+/1tAH19PT0+9fcpUuX1N7errFjx6qgoMDDZQOLRCKqq6vTO++8o6ysrOR7XeFwWKNGjfJ43Z1VVVWpvLxcBQUF6u7uVl1dnZqamtTY2Oj1tAFlZWXd9v7WQw89pHHjxt3X73u9+uqrWr58uQoLC3X16lVVV1crIyNDa9as8XragF555RV95zvf0fbt2/X9739fp06d0p49e7Rnzx6vpw1KIpFQbW2tKioqFAx6/Fe8J1+7ZuDXv/61U1BQ4GRmZjpz5sxxWlpavJ50V++//74j6bajoqLC62kD+rrNkpza2lqvpw1ow4YNTmFhoZOZmemMHz/eWbRokfPXv/7V61kp8cOXKa9evdrJzc11MjMznW9961vO6tWrnYsXL3o9a1D+/Oc/OzNmzHBCoZAzdepUZ8+ePV5PGrTGxkZHknP+/Hmvpzj8uH4AgAnfvwcDALg/ERgAgAkCAwAwQWAAACYIDADABIEBAJggMAAAEwQGAGCCwAAATBAYAIAJAgMAMEFgAAAm/g+MzYfgHN6ELQAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "test = 100\n",
    "img = dataset.images[test]\n",
    "plt.imshow(img, cmap='gray')\n",
    "dataset.target[test]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((1437, 64), (1437, 10), (360, 64), (360, 10))"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = dataset.data\n",
    "Y = dataset.target\n",
    "\n",
    "Y = np.eye(10)[Y]  # Convert Y to one hot\n",
    "\n",
    "x_train, x_test, y_train, y_test = train_test_split(X, Y, test_size=0.2)\n",
    "x_train.shape, y_train.shape, x_test.shape, y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid(x):\n",
    "    return 1 / (1 + np.exp(-x))\n",
    "\n",
    "def softmax(X):\n",
    "    return np.exp(X) / np.sum(np.exp(X))\n",
    "\n",
    "def root_mean_squired_error(Y_pred, Y_test):\n",
    "    return np.sqrt(np.mean((Y_pred - Y_test) ** 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dimension input: 64, Dimension output: 10\n"
     ]
    }
   ],
   "source": [
    "Dimension_input = x_train.shape[1]\n",
    "Dimension_output = y_train.shape[1]\n",
    "\n",
    "H1 = 128\n",
    "H2 = 32\n",
    "\n",
    "lr = .001\n",
    "epochs = 80\n",
    "\n",
    "print(f'Dimension input: {Dimension_input}, Dimension output: {Dimension_output}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "W1, W2, W3 = np.random.randn(Dimension_input, H1), np.random.randn(H1, H2), np.random.randn(H2, Dimension_output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "B1, B2, B3 = np.random.randn(1, H1), np.random.randn(1, H2), np.random.randn(1, Dimension_output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss train: 0.2357576640897113, accuracy train: 0.08907446068197634\n",
      "loss test: 0.23033921533976603, accuracy test: 0.6388888888888888\n",
      "loss train: 0.2220311246675853, accuracy train: 0.0918580375782881\n",
      "loss test: 0.22022679005801735, accuracy test: 0.6888888888888889\n",
      "loss train: 0.21066106308387897, accuracy train: 0.09812108559498957\n",
      "loss test: 0.2121985054371386, accuracy test: 0.7055555555555556\n",
      "loss train: 0.19994524679804976, accuracy train: 0.10160055671537926\n",
      "loss test: 0.20504299030995699, accuracy test: 0.7166666666666667\n",
      "loss train: 0.19018771580389782, accuracy train: 0.09742519137091162\n",
      "loss test: 0.1984758450692517, accuracy test: 0.7416666666666667\n",
      "loss train: 0.18167765706768438, accuracy train: 0.1022964509394572\n",
      "loss test: 0.19151975668161497, accuracy test: 0.7555555555555555\n",
      "loss train: 0.17419608373671586, accuracy train: 0.1057759220598469\n",
      "loss test: 0.18628020683843505, accuracy test: 0.7638888888888888\n",
      "loss train: 0.1674518685016487, accuracy train: 0.10508002783576896\n",
      "loss test: 0.18160594147126385, accuracy test: 0.7694444444444445\n",
      "loss train: 0.1612923988104281, accuracy train: 0.10508002783576896\n",
      "loss test: 0.1778244100254949, accuracy test: 0.7777777777777778\n",
      "loss train: 0.15575125960885916, accuracy train: 0.10508002783576896\n",
      "loss test: 0.17485186370150355, accuracy test: 0.7805555555555556\n",
      "loss train: 0.15066281515449625, accuracy train: 0.10438413361169102\n",
      "loss test: 0.1718827277640057, accuracy test: 0.7944444444444444\n",
      "loss train: 0.14604949820406654, accuracy train: 0.10438413361169102\n",
      "loss test: 0.16884901470992747, accuracy test: 0.7972222222222223\n",
      "loss train: 0.14180446186001333, accuracy train: 0.10368823938761308\n",
      "loss test: 0.16613672360706697, accuracy test: 0.8083333333333333\n",
      "loss train: 0.13802630694004717, accuracy train: 0.10368823938761308\n",
      "loss test: 0.16368023976696602, accuracy test: 0.8111111111111111\n",
      "loss train: 0.1345701062630249, accuracy train: 0.10368823938761308\n",
      "loss test: 0.1613912397806538, accuracy test: 0.8166666666666667\n",
      "loss train: 0.13134914093774736, accuracy train: 0.10368823938761308\n",
      "loss test: 0.15930487939075139, accuracy test: 0.8277777777777777\n",
      "loss train: 0.12833833764261485, accuracy train: 0.10368823938761308\n",
      "loss test: 0.15756195743448287, accuracy test: 0.8388888888888889\n",
      "loss train: 0.1255233166178073, accuracy train: 0.10508002783576896\n",
      "loss test: 0.1560506228278306, accuracy test: 0.8333333333333334\n",
      "loss train: 0.12290504121387374, accuracy train: 0.10438413361169102\n",
      "loss test: 0.15463184669555768, accuracy test: 0.8333333333333334\n",
      "loss train: 0.12045473019014767, accuracy train: 0.10438413361169102\n",
      "loss test: 0.15328689891565464, accuracy test: 0.8472222222222222\n",
      "loss train: 0.11815343072270569, accuracy train: 0.10438413361169102\n",
      "loss test: 0.1520235748621741, accuracy test: 0.8583333333333333\n",
      "loss train: 0.11602090880535001, accuracy train: 0.10368823938761308\n",
      "loss test: 0.15083100645254124, accuracy test: 0.8611111111111112\n",
      "loss train: 0.1140319229286928, accuracy train: 0.10368823938761308\n",
      "loss test: 0.14967470730730337, accuracy test: 0.8638888888888889\n",
      "loss train: 0.1121494117500719, accuracy train: 0.10368823938761308\n",
      "loss test: 0.148536863155788, accuracy test: 0.8666666666666667\n",
      "loss train: 0.11034054460871183, accuracy train: 0.10438413361169102\n",
      "loss test: 0.14741314446928544, accuracy test: 0.8694444444444445\n",
      "loss train: 0.10860988823114408, accuracy train: 0.10368823938761308\n",
      "loss test: 0.1463128509408751, accuracy test: 0.8694444444444445\n",
      "loss train: 0.10695892728233067, accuracy train: 0.10299234516353514\n",
      "loss test: 0.1452472565731218, accuracy test: 0.8694444444444445\n",
      "loss train: 0.10537242688831101, accuracy train: 0.1022964509394572\n",
      "loss test: 0.14423162041637483, accuracy test: 0.8694444444444445\n",
      "loss train: 0.10383090810022029, accuracy train: 0.1022964509394572\n",
      "loss test: 0.14330228782457752, accuracy test: 0.8666666666666667\n",
      "loss train: 0.10230371247036289, accuracy train: 0.1022964509394572\n",
      "loss test: 0.14244581026531267, accuracy test: 0.8666666666666667\n",
      "loss train: 0.10075967128231371, accuracy train: 0.10090466249130133\n",
      "loss test: 0.14157605229863468, accuracy test: 0.8666666666666667\n",
      "loss train: 0.09921784524264324, accuracy train: 0.10160055671537926\n",
      "loss test: 0.14071332826578628, accuracy test: 0.8638888888888889\n",
      "loss train: 0.09732932168453302, accuracy train: 0.10160055671537926\n",
      "loss test: 0.13987087894786088, accuracy test: 0.8694444444444445\n",
      "loss train: 0.09571228446712915, accuracy train: 0.10160055671537926\n",
      "loss test: 0.13907946959913473, accuracy test: 0.8722222222222222\n",
      "loss train: 0.09424450019577116, accuracy train: 0.10160055671537926\n",
      "loss test: 0.13835784414016955, accuracy test: 0.8694444444444445\n",
      "loss train: 0.0929534274858268, accuracy train: 0.10160055671537926\n",
      "loss test: 0.13767418005750087, accuracy test: 0.8722222222222222\n",
      "loss train: 0.09169889811882428, accuracy train: 0.10160055671537926\n",
      "loss test: 0.1370011383156773, accuracy test: 0.875\n",
      "loss train: 0.09046241207207494, accuracy train: 0.10160055671537926\n",
      "loss test: 0.13629581131961255, accuracy test: 0.875\n",
      "loss train: 0.0892532046246608, accuracy train: 0.10160055671537926\n",
      "loss test: 0.13555692393044197, accuracy test: 0.8777777777777778\n",
      "loss train: 0.08808512695211126, accuracy train: 0.10160055671537926\n",
      "loss test: 0.13485304113484692, accuracy test: 0.8777777777777778\n",
      "loss train: 0.08696894784127003, accuracy train: 0.10160055671537926\n",
      "loss test: 0.1341909886027117, accuracy test: 0.8833333333333333\n",
      "loss train: 0.08590343254916369, accuracy train: 0.10160055671537926\n",
      "loss test: 0.13355191630063742, accuracy test: 0.8833333333333333\n",
      "loss train: 0.08487977078520231, accuracy train: 0.10160055671537926\n",
      "loss test: 0.13292388907902009, accuracy test: 0.8833333333333333\n",
      "loss train: 0.08388941470532096, accuracy train: 0.10160055671537926\n",
      "loss test: 0.1323002859992449, accuracy test: 0.8833333333333333\n",
      "loss train: 0.08292641583481644, accuracy train: 0.10160055671537926\n",
      "loss test: 0.13168481643189525, accuracy test: 0.8833333333333333\n",
      "loss train: 0.0819901298393116, accuracy train: 0.10160055671537926\n",
      "loss test: 0.13109542733255725, accuracy test: 0.8861111111111111\n",
      "loss train: 0.08107806613472911, accuracy train: 0.10160055671537926\n",
      "loss test: 0.1305461699598642, accuracy test: 0.8861111111111111\n",
      "loss train: 0.08014709737920782, accuracy train: 0.10160055671537926\n",
      "loss test: 0.13003084696013867, accuracy test: 0.8944444444444445\n",
      "loss train: 0.07916775519431465, accuracy train: 0.10160055671537926\n",
      "loss test: 0.12954864258235882, accuracy test: 0.8972222222222223\n",
      "loss train: 0.07820325813063653, accuracy train: 0.10160055671537926\n",
      "loss test: 0.12913303585415414, accuracy test: 0.8972222222222223\n",
      "loss train: 0.07726945253694159, accuracy train: 0.10160055671537926\n",
      "loss test: 0.12876495324315337, accuracy test: 0.8972222222222223\n",
      "loss train: 0.07637214441582578, accuracy train: 0.10160055671537926\n",
      "loss test: 0.1284148555759177, accuracy test: 0.8944444444444445\n",
      "loss train: 0.07550890764504578, accuracy train: 0.10160055671537926\n",
      "loss test: 0.12806775157899805, accuracy test: 0.8944444444444445\n",
      "loss train: 0.07467045995677517, accuracy train: 0.10160055671537926\n",
      "loss test: 0.12771851859495276, accuracy test: 0.9\n",
      "loss train: 0.07385187172599599, accuracy train: 0.10160055671537926\n",
      "loss test: 0.1273677780024863, accuracy test: 0.9\n",
      "loss train: 0.07305063329741733, accuracy train: 0.1022964509394572\n",
      "loss test: 0.12701783577980558, accuracy test: 0.9027777777777778\n",
      "loss train: 0.07226565221598782, accuracy train: 0.1022964509394572\n",
      "loss test: 0.12666725215019767, accuracy test: 0.9027777777777778\n",
      "loss train: 0.07149260927389371, accuracy train: 0.1022964509394572\n",
      "loss test: 0.12630701587369558, accuracy test: 0.9055555555555556\n",
      "loss train: 0.07071258218382848, accuracy train: 0.1022964509394572\n",
      "loss test: 0.1259134190034966, accuracy test: 0.9055555555555556\n",
      "loss train: 0.06987142483095345, accuracy train: 0.1022964509394572\n",
      "loss test: 0.1254413319110548, accuracy test: 0.9055555555555556\n",
      "loss train: 0.06898429367901622, accuracy train: 0.1022964509394572\n",
      "loss test: 0.12502019164456044, accuracy test: 0.9055555555555556\n",
      "loss train: 0.06813893336205101, accuracy train: 0.1022964509394572\n",
      "loss test: 0.12468173271814327, accuracy test: 0.9055555555555556\n",
      "loss train: 0.06734093089789041, accuracy train: 0.10299234516353514\n",
      "loss test: 0.1244197596396654, accuracy test: 0.9055555555555556\n",
      "loss train: 0.06662175639285943, accuracy train: 0.10299234516353514\n",
      "loss test: 0.12416513210934702, accuracy test: 0.9055555555555556\n",
      "loss train: 0.06593735356775837, accuracy train: 0.10299234516353514\n",
      "loss test: 0.12390630173167524, accuracy test: 0.9027777777777778\n",
      "loss train: 0.06527852147275956, accuracy train: 0.10299234516353514\n",
      "loss test: 0.12364623038767629, accuracy test: 0.9027777777777778\n",
      "loss train: 0.06464425132759948, accuracy train: 0.10368823938761308\n",
      "loss test: 0.1233906823759941, accuracy test: 0.9083333333333333\n",
      "loss train: 0.0640341144677677, accuracy train: 0.10368823938761308\n",
      "loss test: 0.12314273199115255, accuracy test: 0.9083333333333333\n",
      "loss train: 0.06344609526169114, accuracy train: 0.10368823938761308\n",
      "loss test: 0.12290185567466783, accuracy test: 0.9083333333333333\n",
      "loss train: 0.06287745786819314, accuracy train: 0.10368823938761308\n",
      "loss test: 0.12266644483512916, accuracy test: 0.9111111111111111\n",
      "loss train: 0.06232557312462659, accuracy train: 0.10368823938761308\n",
      "loss test: 0.12243532376843015, accuracy test: 0.9111111111111111\n",
      "loss train: 0.06178823311710502, accuracy train: 0.10368823938761308\n",
      "loss test: 0.12220787163768393, accuracy test: 0.9111111111111111\n",
      "loss train: 0.0612637199447658, accuracy train: 0.10368823938761308\n",
      "loss test: 0.12198379745314498, accuracy test: 0.9111111111111111\n",
      "loss train: 0.060750724462258536, accuracy train: 0.10368823938761308\n",
      "loss test: 0.12176296764954975, accuracy test: 0.9111111111111111\n",
      "loss train: 0.060248170127397596, accuracy train: 0.10368823938761308\n",
      "loss test: 0.12154529230778835, accuracy test: 0.9083333333333333\n",
      "loss train: 0.05975501360404576, accuracy train: 0.10368823938761308\n",
      "loss test: 0.12133062746981783, accuracy test: 0.9083333333333333\n",
      "loss train: 0.05927012578478801, accuracy train: 0.10368823938761308\n",
      "loss test: 0.12111868328589584, accuracy test: 0.9083333333333333\n",
      "loss train: 0.05879230739523596, accuracy train: 0.10368823938761308\n",
      "loss test: 0.12090894506148112, accuracy test: 0.9055555555555556\n",
      "loss train: 0.058320419279203964, accuracy train: 0.10368823938761308\n",
      "loss test: 0.1207006419690532, accuracy test: 0.9055555555555556\n",
      "loss train: 0.05785361509305568, accuracy train: 0.10368823938761308\n",
      "loss test: 0.12049287730968476, accuracy test: 0.9055555555555556\n",
      "\n",
      "\n",
      "Train Completed!\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(epochs):\n",
    "\n",
    "    # Train Part\n",
    "    \n",
    "    Y_pred_train = []\n",
    "    for x, y in zip(x_train, y_train):\n",
    "\n",
    "        # Forward Part\n",
    "        x = x.reshape(-1, 1)\n",
    "\n",
    "        # Layer 1\n",
    "        out1 = sigmoid(x.T @ W1 + B1)\n",
    "\n",
    "        # Layer 2\n",
    "        out2 = sigmoid(out1 @ W2 + B2)\n",
    "\n",
    "        # Layer 3\n",
    "        out3 = softmax(out2 @ W3 + B3)\n",
    "\n",
    "        y_pred = out3\n",
    "        Y_pred_train.append(y_pred)\n",
    "\n",
    "        # Back Propagation Part\n",
    "\n",
    "        # Layer 3\n",
    "        error = -2 * (y - y_pred)\n",
    "        grad_W3 = out2.T @ error\n",
    "        grad_B3 = error\n",
    "\n",
    "        # Layer 2\n",
    "        error = error @ W3.T * out2 * (1 - out2)\n",
    "        grad_W2 = out1.T @ error\n",
    "        grad_B2 = error\n",
    "\n",
    "        # Layer 1\n",
    "        error = error @ W2.T * out1 * (1 - out1)\n",
    "        grad_W1 = x @ error\n",
    "        grad_B1 = error\n",
    "\n",
    "        # update\n",
    "\n",
    "        # Layer 1\n",
    "        W1 -= lr * grad_W1\n",
    "        B1 -= lr * grad_B1\n",
    "        \n",
    "        # Layer 2\n",
    "        W2 -= lr * grad_W2\n",
    "        B2 -= lr * grad_B2\n",
    "\n",
    "        # Layer 3\n",
    "        W3 -= lr * grad_W3\n",
    "        B3 -= lr * grad_B3\n",
    "    \n",
    "    # Test Part\n",
    "\n",
    "    Y_pred_test = []\n",
    "    for x, y in zip(x_test, y_test):\n",
    "\n",
    "        x = x.reshape(-1, 1)\n",
    "\n",
    "        out1 = sigmoid(x.T @ W1 + B1)\n",
    "        out2 = sigmoid(out1 @ W2 + B2)\n",
    "        out3 = softmax(out2 @ W3 + B3)\n",
    "\n",
    "        Y_pred_test.append(out3.T)\n",
    "\n",
    "    Y_pred_train = np.array(Y_pred_train).reshape(-1, 10)\n",
    "    Y_pred_test = np.array(Y_pred_test).reshape(-1, 10)\n",
    "\n",
    "    loss_train, loss_test = root_mean_squired_error(Y_pred_train, y_train), root_mean_squired_error(Y_pred_test, y_test)\n",
    "    accuracy_train, accuracy_test = np.mean(np.argmax(Y_pred_train, axis=1) == np.argmax(y_pred, axis=1)), np.mean(np.argmax(Y_pred_test, axis=1) == np.argmax(y_test, axis=1))\n",
    "    \n",
    "    print(f'loss train: {loss_train}, accuracy train: {accuracy_train}')\n",
    "    print(f'loss test: {loss_test}, accuracy test: {accuracy_test}')\n",
    "\n",
    "print('\\n\\nTrain Completed!\\n\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7fa9a1aa89d0>"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZgAAAGdCAYAAAAv9mXmAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAX7klEQVR4nO3df2xVhf3/8ddtu16Y9t4CUqDj0iLDIWIro0BYcU6pmAaJ+gcjBLPyY0sklwESE8M/w2QJl2WZYVtIBWTFxDHczIrOpHSF2RIjhAJpAi5BUJAqArrYe0uTXUzv+fz1uZ9vv8Dlntu+7+GU5yO5ib2e2/OKMTw59/RHwHEcRwAADLECrwcAAIYnAgMAMEFgAAAmCAwAwASBAQCYIDAAABMEBgBggsAAAEwU5fuEqVRKly5dUklJiQKBQL5PDwAYBMdx1Nvbq/LychUUZL5GyXtgLl26pEgkku/TAgCGUHd3tyZOnJjxmLwHpqSkJN+nhI+dOnXK6wk5icfjXk/Iyfz5872eAJ/I5s/yvAeGt8Xghl//QtLf3+/1BMBUNn+Wc5MfAGCCwAAATBAYAIAJAgMAMEFgAAAmCAwAwASBAQCYIDAAABMEBgBggsAAAEwQGACACQIDADBBYAAAJggMAMAEgQEAmCAwAAATOQVm+/btqqys1IgRIzR37lwdO3ZsqHcBAHzOdWDeeustbdy4UZs3b9bJkydVXV2tp556SlevXrXYBwDwKdeBefXVV/WLX/xCK1eu1PTp0/Xaa6/pu9/9rv70pz9Z7AMA+JSrwFy/fl0nTpxQXV3d/32CggLV1dXpyJEjN31NMplUIpEY8AAADH+uAvP111+rv79f48aNG/D8uHHjdPny5Zu+JhaLKRwOpx+RSCT3tQAA3zD/KrJNmzYpHo+nH93d3danBADcAYrcHHzfffepsLBQV65cGfD8lStXNH78+Ju+JhgMKhgM5r4QAOBLrq5giouLNWvWLB06dCj9XCqV0qFDhzRv3rwhHwcA8C9XVzCStHHjRjU0NKimpkZz5szRtm3b1NfXp5UrV1rsAwD4lOvALF26VF999ZV+9atf6fLly3rkkUd04MCBG278AwDubgHHcZx8njCRSCgcDufzlPCxCxcueD0hJz09PV5PyMkjjzzi9QT4RDweVygUyngMP4sMAGCCwAAATBAYAIAJAgMAMEFgAAAmCAwAwASBAQCYIDAAABMEBgBggsAAAEwQGACACQIDADBBYAAAJggMAMAEgQEAmHD9C8eAfCotLfV6Qk78uhsYSlzBAABMEBgAgAkCAwAwQWAAACYIDADABIEBAJggMAAAEwQGAGCCwAAATBAYAIAJAgMAMEFgAAAmCAwAwASBAQCYIDAAABMEBgBggsAAAEwQGACACQIDADDhOjCHDx/W4sWLVV5erkAgoP379xvMAgD4nevA9PX1qbq6Wtu3b7fYAwAYJorcvqC+vl719fUWWwAAw4jrwLiVTCaVTCbTHycSCetTAgDuAOY3+WOxmMLhcPoRiUSsTwkAuAOYB2bTpk2Kx+PpR3d3t/UpAQB3APO3yILBoILBoPVpAAB3GL4PBgBgwvUVzLVr13Tu3Ln0x+fPn1dXV5dGjx6tSZMmDek4AIB/BRzHcdy8oL29XY8//vgNzzc0NGjPnj23fX0ikVA4HHZzStzFenp6vJ5wVyktLfV6AnwiHo8rFAplPMb1FcxPfvITuWwSAOAuxD0YAIAJAgMAMEFgAAAmCAwAwASBAQCYIDAAABMEBgBggsAAAEwQGACACQIDADBBYAAAJggMAMAEgQEAmCAwAAATBAYAYML174MB8smvv3CMX9wFcAUDADBCYAAAJggMAMAEgQEAmCAwAAATBAYAYILAAABMEBgAgAkCAwAwQWAAACYIDADABIEBAJggMAAAEwQGAGCCwAAATBAYAIAJAgMAMEFgAAAmCAwAwISrwMRiMc2ePVslJSUqKyvTs88+qzNnzlhtAwD4mKvAdHR0KBqN6ujRo2pra9O3336rhQsXqq+vz2ofAMCnAo7jOLm++KuvvlJZWZk6Ojr04x//OKvXJBIJhcPhXE+Ju8yFCxe8npCT0tJSryfkxK+7kX/xeFyhUCjjMUWDPYEkjR49+pbHJJNJJZPJ9MeJRGIwpwQA+ETON/lTqZQ2bNig2tpazZgx45bHxWIxhcPh9CMSieR6SgCAj+T8FtmaNWvU0tKiDz74QBMnTrzlcTe7giEyyBZvkeWXX3cj/8zeIlu7dq3ee+89HT58OGNcJCkYDCoYDOZyGgCAj7kKjOM4+uUvf6nm5ma1t7dr8uTJVrsAAD7nKjDRaFR79+7VO++8o5KSEl2+fFmSFA6HNXLkSJOBAAB/cnUPJhAI3PT5pqYmrVixIqvPwZcpww3uweSXX3cj/4b8HswgvmUGAHCX4WeRAQBMEBgAgAkCAwAwQWAAACYIDADABIEBAJggMAAAEwQGAGCCwAAATBAYAIAJAgMAMEFgAAAmCAwAwASBAQCYIDAAABMEBgBggsAAAEwQGACACQIDADBBYAAAJggMAMAEgQEAmCAwAAATBAYAYILAAABMEBgAgAkCAwAwQWAAACYIDADABIEBAJggMAAAEwQGAGCCwAAATBAYAIAJAgMAMOEqMI2NjaqqqlIoFFIoFNK8efPU0tJitQ0A4GOuAjNx4kRt3bpVJ06c0PHjx/XEE0/omWee0UcffWS1DwDgUwHHcZzBfILRo0frt7/9rVavXp3V8YlEQuFweDCnxF3kwoULXk/ISWlpqdcTcuLX3ci/eDyuUCiU8ZiiXD95f3+//va3v6mvr0/z5s275XHJZFLJZDL9cSKRyPWUAAAfcX2T/9SpU7r33nsVDAb1wgsvqLm5WdOnT7/l8bFYTOFwOP2IRCKDGgwA8AfXb5Fdv35dFy9eVDwe19tvv63XX39dHR0dt4zMza5giAyyxVtk+eXX3ci/bN4iG/Q9mLq6Ok2ZMkU7duzI6njuwcANApNfft2N/MsmMIP+PphUKjXgCgUAAMnlTf5Nmzapvr5ekyZNUm9vr/bu3av29na1trZa7QMA+JSrwFy9elU/+9nP9OWXXyocDquqqkqtra168sknrfYBAHzKVWB2795ttQMAMMzws8gAACYIDADABIEBAJggMAAAEwQGAGCCwAAATBAYAIAJAgMAMEFgAAAmCAwAwASBAQCYIDAAABMEBgBggsAAAEwQGACACQIDADDh6heOAfnW09Pj9YSclJaWej0B8BxXMAAAEwQGAGCCwAAATBAYAIAJAgMAMEFgAAAmCAwAwASBAQCYIDAAABMEBgBggsAAAEwQGACACQIDADBBYAAAJggMAMAEgQEAmCAwAAATBAYAYGJQgdm6dasCgYA2bNgwRHMAAMNFzoHp7OzUjh07VFVVNZR7AADDRE6BuXbtmpYvX65du3Zp1KhRQ70JADAM5BSYaDSqRYsWqa6ubqj3AACGiSK3L9i3b59Onjypzs7OrI5PJpNKJpPpjxOJhNtTAgB8yNUVTHd3t9avX68///nPGjFiRFavicViCofD6UckEslpKADAXwKO4zjZHrx//34999xzKiwsTD/X39+vQCCggoICJZPJAf9OuvkVDJFBtrq6uryekJPKykqvJ+SktLTU6wnwiXg8rlAolPEYV2+RLViwQKdOnRrw3MqVKzVt2jS9/PLLN8RFkoLBoILBoJvTAACGAVeBKSkp0YwZMwY8d88992jMmDE3PA8AuLvxnfwAABOuv4rs/9fe3j4EMwAAww1XMAAAEwQGAGCCwAAATBAYAIAJAgMAMEFgAAAmCAwAwASBAQCYIDAAABMEBgBggsAAAEwQGACACQIDADBBYAAAJggMAMAEgQEAmBj0LxwDcKNwOOz1BMBzXMEAAEwQGACACQIDADBBYAAAJggMAMAEgQEAmCAwAAATBAYAYILAAABMEBgAgAkCAwAwQWAAACYIDADABIEBAJggMAAAEwQGAGCCwAAATBAYAIAJAgMAMOEqMK+88ooCgcCAx7Rp06y2AQB8rMjtCx566CEdPHjw/z5BketPAQC4C7iuQ1FRkcaPH2+xBQAwjLi+B3P27FmVl5fr/vvv1/Lly3Xx4sWMxyeTSSUSiQEPAMDw5yowc+fO1Z49e3TgwAE1Njbq/PnzevTRR9Xb23vL18RiMYXD4fQjEokMejQA4M4XcBzHyfXFPT09qqio0KuvvqrVq1ff9JhkMqlkMpn+OJFIEBlkraury+sJOamurvZ6Qk4CgYDXE+AT8XhcoVAo4zGDukNfWlqqBx54QOfOnbvlMcFgUMFgcDCnAQD40KC+D+batWv65JNPNGHChKHaAwAYJlwF5qWXXlJHR4cuXLigDz/8UM8995wKCwu1bNkyq30AAJ9y9RbZ559/rmXLluk///mPxo4dq/nz5+vo0aMaO3as1T4AgE+5Csy+ffusdgAAhhl+FhkAwASBAQCYIDAAABMEBgBggsAAAEwQGACACQIDADBBYAAAJggMAMAEgQEAmCAwAAATBAYAYILAAABMEBgAgAkCAwAwQWAAACYIDADABIEBAJggMAAAEwQGAGCCwAAATBAYAIAJAgMAMEFgAAAmCAwAwASBAQCYIDAAABMEBgBggsAAAEwQGACACQIDADBBYAAAJggMAMAEgQEAmCAwAAATrgPzxRdf6Pnnn9eYMWM0cuRIPfzwwzp+/LjFNgCAjxW5Ofibb75RbW2tHn/8cbW0tGjs2LE6e/asRo0aZbUPAOBTrgLzm9/8RpFIRE1NTennJk+ePOSjAAD+5+otsnfffVc1NTVasmSJysrKNHPmTO3atSvja5LJpBKJxIAHAGD4cxWYTz/9VI2NjZo6dapaW1u1Zs0arVu3Tm+88cYtXxOLxRQOh9OPSCQy6NEAgDtfwHEcJ9uDi4uLVVNTow8//DD93Lp169TZ2akjR47c9DXJZFLJZDL9cSKRIDLIWldXl9cTclJdXe31hJwEAgGvJ8An4vG4QqFQxmNcXcFMmDBB06dPH/Dcgw8+qIsXL97yNcFgUKFQaMADADD8uQpMbW2tzpw5M+C5jz/+WBUVFUM6CgDgf64C8+KLL+ro0aPasmWLzp07p71792rnzp2KRqNW+wAAPuUqMLNnz1Zzc7P+8pe/aMaMGfr1r3+tbdu2afny5Vb7AAA+5er7YCTp6aef1tNPP22xBQAwjPCzyAAAJggMAMAEgQEAmCAwAAATBAYAYILAAABMEBgAgAkCAwAwQWAAACYIDADABIEBAJggMAAAEwQGAGCCwAAATBAYAIAJAgMAMOH6F44B+dTe3u71hJxUVlZ6PQHwHFcwAAATBAYAYILAAABMEBgAgAkCAwAwQWAAACYIDADABIEBAJggMAAAEwQGAGCCwAAATBAYAIAJAgMAMEFgAAAmCAwAwASBAQCYIDAAABMEBgBgwlVgKisrFQgEbnhEo1GrfQAAnypyc3BnZ6f6+/vTH58+fVpPPvmklixZMuTDAAD+5iowY8eOHfDx1q1bNWXKFD322GNDOgoA4H+uAvP/un79ut58801t3LhRgUDglsclk0klk8n0x4lEItdTAgB8JOeb/Pv371dPT49WrFiR8bhYLKZwOJx+RCKRXE8JAPCRnAOze/du1dfXq7y8PONxmzZtUjweTz+6u7tzPSUAwEdyeovss88+08GDB/X3v//9tscGg0EFg8FcTgMA8LGcrmCamppUVlamRYsWDfUeAMAw4TowqVRKTU1NamhoUFFRzl8jAAAY5lwH5uDBg7p48aJWrVplsQcAMEy4vgRZuHChHMex2AIAGEb4WWQAABMEBgBggsAAAEwQGACACQIDADBBYAAAJggMAMAEgQEAmCAwAAATBAYAYILAAABMEBgAgAkCAwAwQWAAACYIDADARN5/JSW/SwZu/Pe///V6Qk4SiYTXEwBT2fxZHnDy/Cf+559/rkgkks9TAgCGWHd3tyZOnJjxmLwHJpVK6dKlSyopKVEgEBjSz51IJBSJRNTd3a1QKDSkn9sSu/OL3fnn1+3svpHjOOrt7VV5ebkKCjLfZcn7W2QFBQW3rd5ghUIhX/3P8L/YnV/szj+/bmf3QOFwOKvjuMkPADBBYAAAJoZVYILBoDZv3qxgMOj1FFfYnV/szj+/bmf34OT9Jj8A4O4wrK5gAAB3DgIDADBBYAAAJggMAMDEsAnM9u3bVVlZqREjRmju3Lk6duyY15Nu6/Dhw1q8eLHKy8sVCAS0f/9+rydlJRaLafbs2SopKVFZWZmeffZZnTlzxutZt9XY2Kiqqqr0N5/NmzdPLS0tXs9ybevWrQoEAtqwYYPXUzJ65ZVXFAgEBjymTZvm9aysfPHFF3r++ec1ZswYjRw5Ug8//LCOHz/u9azbqqysvOG/eSAQUDQa9WTPsAjMW2+9pY0bN2rz5s06efKkqqur9dRTT+nq1ateT8uor69P1dXV2r59u9dTXOno6FA0GtXRo0fV1tamb7/9VgsXLlRfX5/X0zKaOHGitm7dqhMnTuj48eN64okn9Mwzz+ijjz7yelrWOjs7tWPHDlVVVXk9JSsPPfSQvvzyy/Tjgw8+8HrSbX3zzTeqra3Vd77zHbW0tOjf//63fve732nUqFFeT7utzs7OAf+929raJElLlizxZpAzDMyZM8eJRqPpj/v7+53y8nInFot5uModSU5zc7PXM3Jy9epVR5LT0dHh9RTXRo0a5bz++utez8hKb2+vM3XqVKetrc157LHHnPXr13s9KaPNmzc71dXVXs9w7eWXX3bmz5/v9YwhsX79emfKlClOKpXy5Py+v4K5fv26Tpw4obq6uvRzBQUFqqur05EjRzxcdveIx+OSpNGjR3u8JHv9/f3at2+f+vr6NG/ePK/nZCUajWrRokUD/l+/0509e1bl5eW6//77tXz5cl28eNHrSbf17rvvqqamRkuWLFFZWZlmzpypXbt2eT3LtevXr+vNN9/UqlWrhvwHC2fL94H5+uuv1d/fr3Hjxg14fty4cbp8+bJHq+4eqVRKGzZsUG1trWbMmOH1nNs6deqU7r33XgWDQb3wwgtqbm7W9OnTvZ51W/v27dPJkycVi8W8npK1uXPnas+ePTpw4IAaGxt1/vx5Pfroo+rt7fV6WkaffvqpGhsbNXXqVLW2tmrNmjVat26d3njjDa+nubJ//3719PRoxYoVnm3I+09TxvASjUZ1+vRpX7y3Lkk/+MEP1NXVpXg8rrffflsNDQ3q6Oi4oyPT3d2t9evXq62tTSNGjPB6Ttbq6+vT/1xVVaW5c+eqoqJCf/3rX7V69WoPl2WWSqVUU1OjLVu2SJJmzpyp06dP67XXXlNDQ4PH67K3e/du1dfXq7y83LMNvr+Cue+++1RYWKgrV64MeP7KlSsaP368R6vuDmvXrtV7772n999/3/xXMAyV4uJiff/739esWbMUi8VUXV2t3//+917PyujEiRO6evWqfvjDH6qoqEhFRUXq6OjQH/7wBxUVFam/v9/riVkpLS3VAw88oHPnznk9JaMJEybc8BeOBx980Bdv7/2vzz77TAcPHtTPf/5zT3f4PjDFxcWaNWuWDh06lH4ulUrp0KFDvnlv3W8cx9HatWvV3Nysf/3rX5o8ebLXk3KWSqWUTCa9npHRggULdOrUKXV1daUfNTU1Wr58ubq6ulRYWOj1xKxcu3ZNn3zyiSZMmOD1lIxqa2tv+LL7jz/+WBUVFR4tcq+pqUllZWVatGiRpzuGxVtkGzduVENDg2pqajRnzhxt27ZNfX19WrlypdfTMrp27dqAv82dP39eXV1dGj16tCZNmuThssyi0aj27t2rd955RyUlJel7XeFwWCNHjvR43a1t2rRJ9fX1mjRpknp7e7V37161t7ertbXV62kZlZSU3HB/65577tGYMWPu6PteL730khYvXqyKigpdunRJmzdvVmFhoZYtW+b1tIxefPFF/ehHP9KWLVv005/+VMeOHdPOnTu1c+dOr6dlJZVKqampSQ0NDSoq8viPeE++ds3AH//4R2fSpElOcXGxM2fOHOfo0aNeT7qt999/35F0w6OhocHraRndbLMkp6mpyetpGa1atcqpqKhwiouLnbFjxzoLFixw/vnPf3o9Kyd++DLlpUuXOhMmTHCKi4ud733ve87SpUudc+fOeT0rK//4xz+cGTNmOMFg0Jk2bZqzc+dOrydlrbW11ZHknDlzxuspDj+uHwBgwvf3YAAAdyYCAwAwQWAAACYIDADABIEBAJggMAAAEwQGAGCCwAAATBAYAIAJAgMAMEFgAAAmCAwAwMT/AOcotGvUU5HCAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import cv2 as cv\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "image = cv.imread('test_images/test4.png')\n",
    "image = cv.cvtColor(image, cv.COLOR_BGR2GRAY)\n",
    "plt.imshow(image, cmap='gray')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y pred: [[0.26508363 0.99958    0.76564661 0.39855284 0.68909595 0.53594097\n",
      "  0.04866996 0.90924695 0.71917583 0.48228814]]\n",
      "1\n"
     ]
    }
   ],
   "source": [
    "image = image.reshape(64, 1)\n",
    "x = image\n",
    "\n",
    "# forward\n",
    "# layer 1\n",
    "out1 = sigmoid(x.T @ W1 + B1)\n",
    "\n",
    "# layer 2\n",
    "out2 = sigmoid(out1 @ W2 + B2)\n",
    "\n",
    "# layer 3\n",
    "out3 = sigmoid(out2 @ W3 + B3)\n",
    "y_pred = out3\n",
    "\n",
    "print(f'y pred: {y_pred}')\n",
    "print(np.argmax(y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}