MyFFNN.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "36561abf",
   "metadata": {},
   "source": [
    "# Self implemented Feed Forward (Dense) Neural Net (FFNN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "52cfc2be",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Imports\n",
    "import numpy as np\n",
    "import numpy.random as npr\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "8c2ec1f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define different activation functions\n",
    "\n",
    "tanh = np.tanh\n",
    "tanh_derivative = lambda x: 1 - np.tanh(x)**2\n",
    "sigmoid = lambda x: 1/(1+np.exp(-x))\n",
    "sigmoid_derivative = lambda x: -sigmoid(x) * (1-sigmoid(x))\n",
    "relu = np.vectorize(lambda x: x if x>0 else 0)\n",
    "relu_derivative = np.vectorize(lambda x: 1 if x>0 else 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "70e788c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Definition of My dense Neural Network\n",
    "\n",
    "class MyDenseNN:\n",
    "    def __init__(self, shape, learning_rate):\n",
    "        self.shape = shape\n",
    "        self.alist = [np.empty(s) for s in shape]\n",
    "        self.zlist = [np.empty(s) for s in shape] # zlist [0] is unused\n",
    "        \n",
    "        self.agrads = [np.empty(s) for s in shape]\n",
    "        \n",
    "        self.weights = [npr.normal(size=(shape[i],shape[i+1])) for i in range(len(shape)-1)]\n",
    "        self.wgrads = [npr.normal(size=(shape[i],shape[i+1])) for i in range(len(shape)-1)]\n",
    "        self.biases = [npr.normal(size=s) for s in shape[1:]]\n",
    "        self.bgrads = [np.empty(s) for s in shape[1:]]\n",
    "\n",
    "        self.act = tanh\n",
    "        self.act_prime = tanh_derivative\n",
    "        self.loss = np.nan\n",
    "        self.loss_history = []\n",
    "        self.learning_rate = learning_rate\n",
    "\n",
    "    def fProp(self, X):\n",
    "        result = np.empty((len(X), self.shape[-1]))\n",
    "        for i,x in enumerate(X):\n",
    "            result[i] = self.fProp_sample(x)\n",
    "        \n",
    "        return result\n",
    "\n",
    "    def fProp_sample(self, x):\n",
    "        if len(x) != self.shape[0]:\n",
    "            raise Exception(f\"cannot forward propagate with given input len {len(x)}\")\n",
    "\n",
    "        self.alist[0][:] = x\n",
    "        for i in range(len(self.shape)-1):\n",
    "            j = i+1\n",
    "            self.zlist[j][:] = np.dot(self.alist[i], self.weights[i]) + self.biases[i]\n",
    "            self.alist[j][:] = self.act(self.zlist[j])\n",
    "\n",
    "        return self.alist[-1]\n",
    "    \n",
    "    def bProp(self, x, y, y_hat):\n",
    "        self.loss += np.sum((y_hat-y)**2)\n",
    "        \n",
    "        self.agrads[-1][:] = 2*(y_hat - y)\n",
    "        for i in range(len(self.alist))[-2:0:-1]:\n",
    "            j = i+1\n",
    "            self.agrads[i] = np.einsum(\"k,ik\", self.agrads[j]*self.act_prime(self.zlist[j]), self.weights[i])\n",
    "\n",
    "        for i in range(len(self.wgrads))[::-1]:\n",
    "            j = i+1\n",
    "            self.wgrads[i] = np.einsum(\"j,i\", self.agrads[j]*self.act_prime(self.zlist[j]),self.alist[i])\n",
    "            self.bgrads[i] = self.agrads[j]*self.act_prime(self.zlist[j])\n",
    "    \n",
    "    def reset_loss(self):\n",
    "        self.loss = 0\n",
    "    \n",
    "    def fit_sample(self, x, y, add_grads=True):\n",
    "        if add_grads:\n",
    "            self.reset_loss()\n",
    "\n",
    "        y_hat = self.fProp_sample(x)\n",
    "        self.bProp(x,y,y_hat)\n",
    "\n",
    "        if add_grads:\n",
    "            for i in range(len(self.shape)-1):\n",
    "                self.weights[i] -= self.learning_rate*self.wgrads[i]\n",
    "                self.biases[i] -= self.learning_rate*self.bgrads[i]\n",
    "    \n",
    "    def fit_batch(self, X_batch, Y_batch):\n",
    "        for i in range(len(X_batch)):\n",
    "            self.fit_sample(X_batch[i], Y_batch[i], add_grads=False)\n",
    "\n",
    "        for i in range(len(self.shape)-1):\n",
    "            self.weights[i] -= self.learning_rate*self.wgrads[i]\n",
    "            self.biases[i] -= self.learning_rate*self.bgrads[i]\n",
    "        self.loss_history.append(self.loss)\n",
    "        self.reset_loss()\n",
    "        \n",
    "    def fit(self,X,Y,batchsize,epochs=1):\n",
    "        assert len(X) == len(Y), f\"data and label should have same length but have length {len(X)} and {len(Y)}\"\n",
    "        rnd_idx = np.arange(len(X)); npr.shuffle(rnd_idx)\n",
    "        for ep in range(epochs):\n",
    "#             print(f\"epoch: {ep+1}/{epochs}\")\n",
    "            for b in range(len(X)//batchsize):\n",
    "                start = b*batchsize\n",
    "                end = (b+1)*batchsize\n",
    "                end = end if end <= len(X) else len(X)\n",
    "                batch_idx = rnd_idx[start:end]\n",
    "                self.fit_batch(X[batch_idx], Y[batch_idx])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "42439457",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "out before training:  [[-0.36868612 -0.10918944]]\n",
      "out after training:  [[0.5 0.4]]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY8AAAEGCAYAAACdJRn3AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAamklEQVR4nO3df7BfdX3n8efr+/0mCIrBxkzFhDbpkorRTsVGVqvbcYqtob/i7uAaut0yXXbZ7kKrnbod6I66S5edsu0U64jOMEJF1hHY1G3TDlu6Lbqt3QoJSFWIGVOgTShqBETEheQm7/3jnHvv997c5HvuJd9cuff5mMnwPZ/v55zvOfeE+8r5vM/5fFNVSJI0H73F3gFJ0vOP4SFJmjfDQ5I0b4aHJGneDA9J0rwNFnsHTpaXvvSltX79+sXeDUl6Xrnnnnu+XlVrZrePPTySbAF+F+gDH6mq35z1/inAx4AfAh4D3lFVDydZDWwHXgd8tKoub/ufBvwP4B8Bh4E/qqorRu3H+vXr2bVr14k7MElaBpL83VztYx22StIHrgMuADYBFyXZNKvbJcATVXU2cC1wTdv+DPAe4N1zbPq3q+oc4FzgjUkuGMf+S5LmNu6ax3nA3qp6sKoOArcAW2f12Qrc1L7eDpyfJFX1dFV9hiZEplTVt6vqU+3rg8C9wLpxHoQkaaZxh8daYN/Q8v62bc4+VTUBPAms7rLxJGcAPw38+THevzTJriS7Dhw4ML89lyQd0/P2bqskA+ATwAeq6sG5+lTV9VW1uao2r1lzVL1HkrRA4w6PR4CzhpbXtW1z9mkDYRVN4XyU64EvV9X7n/tuSpLmY9zhsRPYmGRDkpXANmDHrD47gIvb1xcCd9aI2RqT/BeakHnXid1dSVIXY71Vt6omklwO3EFzq+6NVXV/kquAXVW1A7gBuDnJXuBxmoABIMnDwIuBlUneBvw48E3gPwJfAu5NAvDBqvrIOI9FkjRt7M95VNXtwO2z2t479PoZ4O3HWHf9MTabE7V/ozw7cZgqeMGK/sn6SEn6jve8LZifLD/5gc/wq7f9zWLvhiR9RzE8Rhj0wsSRI4u9G5L0HcXwGKHfCxOH/bZFSRpmeIww6PeYOGJ4SNIww2OEQS8cNjwkaQbDY4R+Lxw6bM1DkoYZHiOs6HvlIUmzGR4j9HvWPCRpNsNjBG/VlaSjGR4jDLxVV5KOYniMMLDmIUlHMTxGsOYhSUczPEZYYc1Dko5ieIzg9CSSdDTDY4RBPw5bSdIshscIg17PgrkkzWJ4jOD0JJJ0NMNjBCdGlKSjGR4jOCW7JB3N8BihecLcYStJGmZ4jNDvhSMFR7z6kKQphscIK/oB4HAZHpI0yfAYod9rfkQ+KChJ0wyPEQa95srDKUokaZrhMcJgctjKmockTTE8Rpi88jjksJUkTTE8RpiseXjlIUnTxh4eSbYk2ZNkb5Ir5nj/lCS3tu/flWR92746yaeSfCvJB2et80NJvtCu84EkGdf+Tw5bWfOQpGljDY8kfeA64AJgE3BRkk2zul0CPFFVZwPXAte07c8A7wHePcemPwz8G2Bj+2fLid/7xlTB3GErSZoy7iuP84C9VfVgVR0EbgG2zuqzFbipfb0dOD9JqurpqvoMTYhMSXIm8OKq+mxVFfAx4G3jOoBBv71V12ErSZoy7vBYC+wbWt7fts3Zp6omgCeB1SO2uX/ENgFIcmmSXUl2HThwYJ673pi88rDmIUnTlnTBvKqur6rNVbV5zZo1C9pGf+puK2sekjRp3OHxCHDW0PK6tm3OPkkGwCrgsRHbXDdimyfMCp/zkKSjjDs8dgIbk2xIshLYBuyY1WcHcHH7+kLgzraWMaeqehT4ZpLXt3dZ/Tzwhyd+1xtT05MYHpI0ZTDOjVfVRJLLgTuAPnBjVd2f5CpgV1XtAG4Abk6yF3icJmAASPIw8GJgZZK3AT9eVQ8A/x74KHAq8L/aP2MxfbeVw1aSNGms4QFQVbcDt89qe+/Q62eAtx9j3fXHaN8FvPrE7eWxWTCXpKMt6YL5iTD9kKDhIUmTDI8RpmseDltJ0iTDYwSfMJekoxkeIzgluyQdzfAYYWpKdsNDkqYYHiMMpqZkt+YhSZMMjxH61jwk6SiGxwjeqitJRzM8Rhg4PYkkHcXwGMHpSSTpaIbHCH1v1ZWkoxgeI6xw2EqSjmJ4jNB32EqSjmJ4jDBV8/DKQ5KmGB4j9HqhF2sekjTM8Ohg0OtxyIcEJWmK4dFBvxenJ5GkIYZHB4N+rHlI0hDDo4NBL85tJUlDDI8O+r2eVx6SNMTw6GBF35qHJA0zPDroO2wlSTMYHh0MehbMJWmY4dHBoN/zIUFJGmJ4dDDohUPObSVJUwyPDpqHBL3ykKRJYw+PJFuS7EmyN8kVc7x/SpJb2/fvSrJ+6L0r2/Y9Sd461P4rSe5P8sUkn0jygnEew6DvrbqSNGys4ZGkD1wHXABsAi5KsmlWt0uAJ6rqbOBa4Jp23U3ANuBVwBbgQ0n6SdYCvwxsrqpXA/2239g0BXOHrSRp0rivPM4D9lbVg1V1ELgF2Dqrz1bgpvb1duD8JGnbb6mqZ6vqIWBvuz2AAXBqkgFwGvAP4zwIb9WVpJnGHR5rgX1Dy/vbtjn7VNUE8CSw+ljrVtUjwG8Dfw88CjxZVX86lr1vNQ8JGh6SNOl5VzBP8hKaq5INwMuBFyb5uWP0vTTJriS7Dhw4sODP7Pd6HDI8JGnKuMPjEeCsoeV1bducfdphqFXAY8dZ9y3AQ1V1oKoOAZ8EfniuD6+q66tqc1VtXrNmzYIPYoVTskvSDOMOj53AxiQbkqykKWzvmNVnB3Bx+/pC4M6qqrZ9W3s31gZgI3A3zXDV65Oc1tZGzgd2j/MgrHlI0kyDcW68qiaSXA7cQXNX1I1VdX+Sq4BdVbUDuAG4Ocle4HHaO6fafrcBDwATwGVVdRi4K8l24N62/XPA9eM8Dr/PQ5JmGmt4AFTV7cDts9reO/T6GeDtx1j3auDqOdrfB7zvxO7psQ16Tk8iScOedwXzxeBzHpI0k+HRgTUPSZrJ8OjA6UkkaSbDo4OBEyNK0gyGRwd9p2SXpBkMjw6cnkSSZjI8Ouj3ehbMJWmI4dGBt+pK0kyGRweDfjhScMShK0kCDI9OBr0AeLuuJLUMjw76vebHZNFckhqGRwcr+pNXHtY9JAkMj076k8NW3nElSYDh0Yk1D0mayfDoYNC35iFJwwyPDiaHrZyiRJIahkcHk8NWXnlIUsPw6GBy2MqahyQ1DI8OpgvmDltJEhgenXirriTNZHh0MPmQoDUPSWoYHh1MTk/isJUkNQyPDgYOW0nSDIZHB96qK0kzGR4dDNqaxyHDQ5IAw6OT6SnZrXlIEiwwPJKcc6J35DuZNQ9JmmlkeCT5D0n+Oskrh5ofSfKLXT4gyZYke5LsTXLFHO+fkuTW9v27kqwfeu/Ktn1PkrcOtZ+RZHuSLyXZneQNXfZloQZ9Z9WVpGFdrjzOBt4FHJhsqKqngJ8etWKSPnAdcAGwCbgoyaZZ3S4Bnqiqs4FrgWvadTcB24BXAVuAD7XbA/hd4E+q6hzgB4HdHY5jwZySXZJm6hIedwJvAg5ONiR5KfDGDuueB+ytqger6iBwC7B1Vp+twE3t6+3A+UnStt9SVc9W1UPAXuC8JKuAHwFuAKiqg1X1jQ77smADax6SNMPI8KiqW9t+f5tkZ5KrgR8G9nTY/lpg39Dy/rZtzj5VNQE8Caw+zrobaK6Cfi/J55J8JMkL5/rwJJcm2ZVk14EDB+bq0sn0lOxeeUgSdCyYV9VvAd8DvA/oA+8Gnhrjfh3PAHgt8OGqOhd4GjiqlgJQVddX1eaq2rxmzZqFf6DTk0jSDIOuHavq/wG3t39I8uYOqz0CnDW0vK5tm6vP/iQDYBXw2HHW3Q/sr6q72vbtHCM8TpRBzynZJWnYgp/zqKpPd+i2E9iYZEOSlTQF8B2z+uwALm5fXwjcWVXVtm9r78baAGwE7q6qrwD7kryiXed84IGFHkcX07fqWvOQJJjHlcdCVNVEksuBO2iGu26sqvuTXAXsqqodNIXvm5PsBR6nCRjafrfRBMMEcFlVHW43/UvAx9tAehD4hXEeh8NWkjTTWMMDoKqmhrqG2t479PoZ4O3HWPdq4Oo52u8DNp/QHT0Oh60kaSanJ+mg77CVJM1geHTgQ4KSNJPh0UGvF3pxbitJmmR4dDTo9bzykKSW4dHRoB+nJ5GkluHRUb8XpyeRpJbh0dGgF5/zkKSW4dHRoG/NQ5ImGR4dDXrxOQ9JahkeHfUdtpKkKYZHRysctpKkKYZHR/1emPBWXUkCDI/OmpqHVx6SBIZHZ81DgoaHJIHh0Vm/1+OQ4SFJgOHRWfOQoDUPSQLDozNrHpI0zfDoaNCPt+pKUsvw6KjvlOySNMXw6GiFNQ9JmmJ4dNS35iFJUwyPjqx5SNI0w6OjQa/nQ4KS1DI8Ohr0wiGnZJckwPDozCnZJWma4dGR3yQoSdMMj478JkFJmjb28EiyJcmeJHuTXDHH+6ckubV9/64k64feu7Jt35PkrbPW6yf5XJI/HvcxwOT3eXjlIUkw5vBI0geuAy4ANgEXJdk0q9slwBNVdTZwLXBNu+4mYBvwKmAL8KF2e5PeCewe5/4PW+GU7JI0ZdxXHucBe6vqwao6CNwCbJ3VZytwU/t6O3B+krTtt1TVs1X1ELC33R5J1gE/CXxkzPs/pd/r+ZCgJLXGHR5rgX1Dy/vbtjn7VNUE8CSwesS67wd+DThuESLJpUl2Jdl14MCBBR5CY+DX0ErSlOddwTzJTwFfq6p7RvWtquuranNVbV6zZs1z+txBPxwpOOLQlSSNPTweAc4aWl7Xts3ZJ8kAWAU8dpx13wj8TJKHaYbBfjTJfx/Hzg8b9AJg0VySGH947AQ2JtmQZCVNAXzHrD47gIvb1xcCd1ZVte3b2ruxNgAbgbur6sqqWldV69vt3VlVPzfm46Dfa35UFs0lCQbj3HhVTSS5HLgD6AM3VtX9Sa4CdlXVDuAG4OYke4HHaQKBtt9twAPABHBZVR0e5/4ez4p+c+Vx6MgRTqU/orckLW1jDQ+AqroduH1W23uHXj8DvP0Y614NXH2cbX8a+PSJ2M9R+u2w1WHvuJKk51/BfLEM+s2PypqHJBkenU0XzL1dV5IMj44mh618UFCSDI/OJgvm3m0lSYZHZ5O36jpsJUmGR2c+JChJ0wyPjgbWPCRpiuHR0aDvlYckTTI8OpqensSahyQZHh2tcNhKkqYYHh31LZhL0hTDoyNrHpI0zfDoaGDNQ5KmGB4dTQ5bHbLmIUmGR1cDpyeRpCmGR0eDnlOyS9Ikw6Oj6SfMrXlIkuHRkbfqStI0w6OjFf3Ju60MD0kyPDrqO2wlSVMMj46ckl2SphkeHXmrriRNMzw6mrxV14cEJcnw6Gyy5uH0JJJkeHRmzUOSphkeHfV6oRe/z0OS4CSER5ItSfYk2ZvkijnePyXJre37dyVZP/TelW37niRvbdvOSvKpJA8kuT/JO8d9DJMGvZ5XHpLEmMMjSR+4DrgA2ARclGTTrG6XAE9U1dnAtcA17bqbgG3Aq4AtwIfa7U0Av1pVm4DXA5fNsc2xGPRjzUOSGP+Vx3nA3qp6sKoOArcAW2f12Qrc1L7eDpyfJG37LVX1bFU9BOwFzquqR6vqXoCqegrYDawd83EATdHcu60kafzhsRbYN7S8n6N/0U/1qaoJ4ElgdZd12yGuc4G75vrwJJcm2ZVk14EDBxZ+FK1BLz7nIUk8jwvmSV4E/D7wrqr65lx9qur6qtpcVZvXrFnznD9z0O8x4bCVJI09PB4BzhpaXte2zdknyQBYBTx2vHWTrKAJjo9X1SfHsudzGPTi3VaSxPjDYyewMcmGJCtpCuA7ZvXZAVzcvr4QuLOqqm3f1t6NtQHYCNzd1kNuAHZX1e+Mef9n6DtsJUkADMa58aqaSHI5cAfQB26sqvuTXAXsqqodNEFwc5K9wOM0AUPb7zbgAZo7rC6rqsNJ3gT8S+ALSe5rP+rXq+r2cR4LNNOyHzI8JGm84QHQ/lK/fVbbe4dePwO8/RjrXg1cPavtM0BO/J6O1lx5WPOQpOdtwXwxWPOQpIbhMQ+DfnzCXJIwPOal7/QkkgQYHvOywpqHJAGGx7w4PYkkNQyPeVjR73FwwisPSTI85mHdS07l7x57muYZRklavgyPeTjnZafzxLcP8dVvPrvYuyJJi8rwmIdXnvliAHZ/Zc55GCVp2TA85uGclzXh8aVHn1rkPZGkxWV4zMOq01aw9oxT2f2oVx6SljfDY57OednpfMlhK0nLnOExT+eceTp/e+Bpnp04vNi7IkmLxvCYp1ee+WIOHym+/NVvLfauSNKiMTzmaapo/hWL5pKWL8NjntavPo1TBj2+ZNFc0jJmeMzToN/jFS873Wc9JC1rhscCnPOy09n96FNOUyJp2TI8FuCVZ76Yx58+yIFvOU2JpOXJ8FiAyaL5bp80l7RMGR4L8MozTwewaC5p2TI8FuCM01Zy5qoXeLuupGXL8Figc152Ovft+wZH/E5zScuQ4bFAW1+zloe+/jQf++uHF3tXJOmkMzwWaOtrXs6bX7GG/3bHHvY9/u3F3h1JOqkMjwVKwn/9pz9AL+GKT37eZz4kLSuGx3Pw8jNO5cqfOIe/2vsYt+7ct9i7I0knzdjDI8mWJHuS7E1yxRzvn5Lk1vb9u5KsH3rvyrZ9T5K3dt3myXTR676HN3zfat7zh1/kyk9+noe//vRi7o4knRSDcW48SR+4DvgxYD+wM8mOqnpgqNslwBNVdXaSbcA1wDuSbAK2Aa8CXg78WZLvb9cZtc2TptcLH/zZc3n/n32ZW3ft49ad+3jLK7+bH1i7iu9b8yK+d/VprDp1BS88ZcBpK/us6PfopRn2kqTnq7GGB3AesLeqHgRIcguwFRj+Rb8V+E/t6+3AB9P8Zt0K3FJVzwIPJdnbbo8O2zypVr/oFH7jba/ml84/mxs+8xB/dN8/8KcPfPW46wx6oZeQ0PxhOkyGc8WIkfRc3fOeH+MFK/ondJvjDo+1wHAxYD/wj4/Vp6omkjwJrG7bPztr3bXt61HbBCDJpcCl7eK3kuyZx76/FPj6PPovFR738uJxLwOn/sbUy4Uc9/fO1Tju8FhUVXU9cP1C1k2yq6o2n+Bd+o7ncS8vHvfyciKPe9wF80eAs4aW17Vtc/ZJMgBWAY8dZ90u25QkjdG4w2MnsDHJhiQraQrgO2b12QFc3L6+ELizmocmdgDb2ruxNgAbgbs7blOSNEZjHbZqaxiXA3cAfeDGqro/yVXArqraAdwA3NwWxB+nCQPafrfRFMIngMuq6jDAXNscw+4vaLhrCfC4lxePe3k5Yccdn4yWJM2XT5hLkubN8JAkzZvhMct30tQn45TkrCSfSvJAkvuTvLNt/64k/zvJl9v/vmSx93UckvSTfC7JH7fLG9rpcfa20+WsXOx9HIckZyTZnuRLSXYnecNyOOdJfqX9e/7FJJ9I8oKleM6T3Jjka0m+ONQ25/lN4wPt8X8+yWvn81mGx5Ch6VQuADYBF7XTpCxFE8CvVtUm4PXAZe2xXgH8eVVtBP68XV6K3gnsHlq+Bri2qs4GnqCZNmcp+l3gT6rqHOAHaX4GS/qcJ1kL/DKwuapeTXOjzeRUSEvtnH8U2DKr7Vjn9wKau1g30jxM/eH5fJDhMdPUdCpVdRCYnPpkyamqR6vq3vb1UzS/RNbSHO9NbbebgLctyg6OUZJ1wE8CH2mXA/wozfQ4sHSPexXwIzR3OFJVB6vqGyyDc05zZ+mp7bNkpwGPsgTPeVX9Bc1dq8OOdX63Ah+rxmeBM5Kc2fWzDI+Z5ppOZe0x+i4Z7UzG5wJ3Ad9dVY+2b30F+O7F2q8xej/wa8CRdnk18I2qmmiXl+p53wAcAH6vHbL7SJIXssTPeVU9Avw28Pc0ofEkcA/L45zDsc/vc/p9Z3gsc0leBPw+8K6q+ubwe+3DmkvqXu4kPwV8raruWex9WQQD4LXAh6vqXOBpZg1RLdFz/hKaf2VvoJmh+4UcPbSzLJzI82t4zLSspj5JsoImOD5eVZ9sm786eena/vdri7V/Y/JG4GeSPEwzLPmjNHWAM9ohDVi6530/sL+q7mqXt9OEyVI/528BHqqqA1V1CPgkzd+D5XDO4djn9zn9vjM8Zlo2U5+04/w3ALur6neG3hqeLuZi4A9P9r6NU1VdWVXrqmo9zfm9s6r+BfApmulxYAkeN0BVfQXYl+QVbdP5NDM4LOlzTjNc9fokp7V/7yePe8mf89axzu8O4Ofbu65eDzw5NLw1kk+Yz5LkJ2jGxCenPrl6cfdoPJK8CfhL4AtMj/3/Ok3d4zbge4C/A/55Vc0uwC0JSd4MvLuqfirJ99FciXwX8Dng59rvkllSkryG5kaBlcCDwC/Q/CNySZ/zJP8ZeAfNXYafA/41zfj+kjrnST4BvJlm6vWvAu8D/oA5zm8bpB+kGcL7NvALVbWr82cZHpKk+XLYSpI0b4aHJGneDA9J0rwZHpKkeTM8JEnzZnhI85TkW+1/1yf52RO87V+ftfx/T+T2pRPF8JAWbj0wr/AYeqL5WGaER1X98Dz3STopDA9p4X4T+CdJ7mu/L6Kf5LeS7Gy/H+HfQvMwYpK/TLKD5slmkvxBknva75i4tG37TZqZX+9L8vG2bfIqJ+22v5jkC0neMbTtTw99R8fH24e/pLEa9a8gScd2Be0T6gBtCDxZVa9LcgrwV0n+tO37WuDVVfVQu/yv2qd8TwV2Jvn9qroiyeVV9Zo5PuufAa+h+Q6Ol7br/EX73rnAq4B/AP6KZt6mz5zog5WGeeUhnTg/TjNX0H0007yspvmiHYC7h4ID4JeT/A3wWZrJ6TZyfG8CPlFVh6vqq8D/AV43tO39VXUEuI9mOE0aK688pBMnwC9V1R0zGps5tJ6etfwW4A1V9e0knwZe8Bw+d3g+psP4/7VOAq88pIV7Cjh9aPkO4N+1U92T5PvbL1uabRXwRBsc59B8DfCkQ5Prz/KXwDvausoamm8EvPuEHIW0AP4LRVq4zwOH2+Gnj9J8L8h64N62aH2Aub/a9E+AX0yyG9hDM3Q16Xrg80nubaeKn/Q/gTcAf0PzZT6/VlVfacNHOumcVVeSNG8OW0mS5s3wkCTNm+EhSZo3w0OSNG+GhyRp3gwPSdK8GR6SpHn7/13XNOMi2pJNAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Test NN\n",
    "\n",
    "sample = npr.rand(5).reshape(1,5)\n",
    "out = np.array([0.5,0.4]).reshape(1,2)\n",
    "iters = 100\n",
    "loss = np.empty(iters)\n",
    "\n",
    "nn = MyDenseNN([5,3,2], learning_rate=0.1)\n",
    "print(\"out before training: \", nn.fProp(sample))\n",
    "\n",
    "nn.fit(sample, out, batchsize=1, epochs=iters)\n",
    "# for i in range(iters):\n",
    "#     loss[i] = nn.loss\n",
    "    \n",
    "\n",
    "print(\"out after training: \", nn.fProp(sample))\n",
    "\n",
    "plt.plot(range(len(nn.loss_history)), nn.loss_history)\n",
    "plt.ylim(0,None)\n",
    "plt.xlabel(\"Iteration\")\n",
    "plt.ylabel(r\"$\\mathcal{L}$\")\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4d80a0f0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPkAAAD8CAYAAAC1ggIjAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAATI0lEQVR4nO3df6xfdX3H8efLUtBNhEINVEB+xMaJuineIGhiOoFYiKMkYgJLpBhI1UnmFk1WJYGEZBuaTDMGkzTQAIYAGxq8LnUEBIaLgVFJoRQCFJaF1kpt64oNCNx7X/vje2q+u37v93xvz7nf7/mevh7m5J7v93w8n3cvfffzOZ/zOZ8j20REe71l1AFExMJKkke0XJI8ouWS5BEtlySPaLkkeUTLVUpySUdJuk/S88XPJXOUm5a0qdgmq9QZEfOjKvfJJX0L2GP7WklrgSW2/6ZHuX22314hzog4QFWT/Flghe0dkpYBD9l+b49ySfKIEama5P9r+8hiX8Cv93+eVW4K2ARMAdfavmeO860B1gAsftuijyw56R0HHFvbHbno1VGH0HjPPfX6LtvvrHKOT/3pH3r3numByv78ydfvtb2ySn0L4ZCyApLuB47tcejK7g+2LWmufzFOtL1d0inAA5I2235hdiHb64B1AMecepT//PZzSv8AB6s/O3LTqENovLNOee5/qp5j155pHr33+IHKLl72wtKq9S2E0iS3ffZcxyS9LGlZV3d95xzn2F78fFHSQ8CHgd9L8ojmMdOeGXUQlVS9hTYJrC72VwM/nF1A0hJJhxX7S4GPA09XrDdiKAzM4IG2pqqa5NcC50h6Hji7+IykCUk3FWXeB2yU9ATwIJ1r8iR5jI2ZAf9XRtJ6STslPTXH8RWS9nbdbr6qjvhLu+v92N4NnNXj+43A5cX+z4APVqknYlSMebO+7votwPXAbX3K/NT2p+uqEComeUTbGZiuqStu+2FJJ9VysnnItNaIEvO4Jl8qaWPXtuYAqjtT0hOSfizp/XXEn5Y8og8D04PPJdlle6JCdY/Tud28T9J5wD3A8grnA9KSR5SaGXCryvYrtvcV+xuAxcUdqUrSkkf0YVzbNXkZSccCLxcTy06n0wjvrnreJHlEHza8WVOOS7oDWEHn2n0bcDWwuFOPbwQuBL5UTAN/DbjINay0miSP6EtMo1rOZPvikuPX07nFVqskeUQfBmaaO5ltIEnyiBJ1teSjkiSP6KMzGSZJHtFaBt70eN9pTpJH9GHE9JhPJ0mSR5SYcbrrEa2Va/KI1hPTuSaPaK/OyjBJ8ojWssUbXjTqMCpJkkeUmMk1eUR7dQbe0l2PaLEMvEW0WgbeIg4C05kME9FeRrzp8U6T8Y4+YoFl4C2i5YzSXY9ouwy8RbSYzdjfQqslekkrJT0raauktT2OHybpruL4o6N4VUzEgegMvC0aaGuqykkuaRFwA3AucCpwsaRTZxW7DPi17fcA3wG+WbXeiGGZ5i0DbU1VR2SnA1ttv2j7DeBOYNWsMquAW4v9u4GzJI33aEYcFIyY8WBbU9WR5McBL3V93lZ817OM7SlgL3B0DXVHLLhxb8kbNfBWvAVyDcDhx/7BiKOJ2L/uenMTeBB1RL8dOKHr8/HFdz3LSDoEOIIe73iyvc72hO2Jty05rIbQIqrqvEFlkK2p6kjyx4Dlkk6WdChwETA5q8wksLrYvxB4oI53PEUstM6SzOM9ul65u257StIVwL3AImC97S2SrgE22p4Ebga+J2krsIfOPwQRjWdr7LvrtVyTF+9S3jDru6u69n8LfLaOuiKGbdwnwzRq4C2iaTrPkzf3ensQSfKIvrIyTESrdW6hpSWPaK39c9fHWZI8okQeNY1osc6jpuPdXR/vf6IihqCuB1QkrZe0U9JTcxyXpOuKR7KflHRaHfEnySP66DyF9paBtgHcAqzsc/xcYHmxrQG+W/kPQLrrEX11prXW0xbafrhkwZRVwG3FlO9HJB0paZntHVXqTZJH9DXUaa1zPbadJI9YSPOY8bZU0sauz+tsr1uAkOYlSR7RxzxH13fZnqhQ3SCPbc9bBt4iStQ48FZmErikGGU/A9hb9Xoc0pJH9LV/jbc6SLoDWEGnW78NuBpYDGD7RjpPcp4HbAVeBT5fR71J8og+DEzVN7p+cclxA1+upbIuSfKIElk0IqLNGr7c8iCS5BF9ZNGIiINAWvKIFsuiEREtZ8TUTAbeIlot1+QRbeZ01yNaLdfkEQeBJHlEixkxnYG3iHbLwFtEi7kFA2+19EMkrZT0bLHK5Noexy+V9CtJm4rt8jrqjRgGWwNtTVW5JZe0CLgBOIfOmlSPSZq0/fSsonfZvqJqfRHDNf4PqNTRkp8ObLX9ou03gDvprDoZ0QoHfUtO7xUmP9qj3GckfQJ4Dvhr2y/NLiBpDZ31pnn3cYfwD8seryG8dvrUuz406hDGwHOVz2DD9ExzE3gQw7o38CPgJNt/DNwH3NqrkO11tidsT7zz6PF+yVy0xwwaaGuqOpK8dIVJ27ttv158vAn4SA31Riw4M/7d9TqS/DFguaSTJR0KXERn1cnfkbSs6+P5wDM11BsxBIO9B63Jg3OVr8ltT0m6ArgXWASst71F0jXARtuTwF9KOh+YAvYAl1atN2JY7FFHUE0tk2Fsb6CznGz3d1d17X8d+HoddUUMW5O74oPIjLeIPjqj65m7HtFq6a5HtFy66xEtZpp9e2wQSfKIEmPeW0+SR/Rl8JhPa02SR5RIdz2i5TK6HtFi++euj7MkeUQ/BpLkEe2W7npEqymj6xGtl5Y8osU8/gNv4/14TcQweMBtAKNYvjwteUSpelryUS1fnpY8oszMgFu5kSxfniSP6Gf/ffJBNlgqaWPXtmbW2XotX35cj1o/I+lJSXdLOqHH8XlJdz2ixDzuk++yPVGxuh8Bd9h+XdIX6Cxf/skqJ0xLHlGmvoG3kSxfniSPKDN4d73MSJYvT3c9ooRqmgwzquXLk+QR/VhQ47TWUSxfniSPKJNprREtlySPaLkkeUSLtWDRiFpuoUlaL2mnpKfmOC5J1xWT8p+UdFod9UYMgzzY1lR13Se/BVjZ5/i5wPJiWwN8t6Z6IxZejU+hjUItSW77YTr39OayCrjNHY8AR8666R/RWGnJBzPQxHxJa/ZP7v/V7ukhhRZRor4ZbyPRqGmtttfZnrA98c6jF406nIjBu+oNbsmHNbpeOjE/orEanMCDGFZLPglcUoyynwHstb1jSHVHVKKZwbamqqUll3QHsILOQ/PbgKuBxQC2b6QzV/c8YCvwKvD5OuqNGIoxb8lrSXLbF5ccN/DlOuqKGKamj5wPIjPeIso0eOR8EEnyiDJpySPaLd31iDZzs0fOB5EkjyiTljyi5ZLkEe027tfkjZq7HhH1S0seUWbMW/IkeUQ/GV2POAikJY9oLzH+A29J8ogySfKIFstTaBEHgQy8RbRbWvKItkuSR7RYw1diHUSSPKJEuusRbZckj2i3TGuNaLMWXJPnUdOIPjSPbaDzSSslPVu8xnttj+OHSbqrOP6opJOq/hmS5BFlanoXmqRFwA10XuV9KnCxpFNnFbsM+LXt9wDfAb5ZNfwkeUSJGl9dfDqw1faLtt8A7qTzWu9uq4Bbi/27gbMkVVr4PUkeUWbwlnzp/ldvF9uaWWca5BXevytjewrYCxxdJfwMvEX0M79FI3bZnljAaA5ILS25pPWSdkp6ao7jKyTtlbSp2K6qo96Ioajv/eSDvML7d2UkHQIcAew+4Nipr7t+C7CypMxPbX+o2K6pqd6IBVfjNfljwHJJJ0s6FLiIzmu9u00Cq4v9C4EHiheGHrC63mr6cB1D/RGNVNN9cttTkq4A7gUWAettb5F0DbDR9iRwM/A9SVuBPXT+IahkmNfkZ0p6AvgF8DXbW2YXKAYq1gAsfdeh/Mu+I4YY3nj57787c9QhNN/X767lNHXOXbe9Adgw67uruvZ/C3y2vhqHN7r+OHCi7T8B/gm4p1ch2+tsT9ieOPyojAlGA5jOohGDbA01lCS3/YrtfcX+BmCxpKXDqDuiiv0LOdZ0TT4SQ0lyScfuv6Ev6fSi3kojhhFDU9/o+kjU0ieWdAewgs5kgG3A1cBiANs30hkl/JKkKeA14KKqI4YRw6Ix/6ta1+j6xSXHrweur6OuiKFqeCs9iIxuRZRo8vX2IJLkESWyaERE26Ulj2ixht8eG0SSPKJMkjyivfJW04iDgGbGO8uT5BH95D55RPvlFlpE26Ulj2i3DLxFtJmBPKAS0W65Jo9osdwnj2g7O931iLZLSx7RdknyiHZLSx7RZgamxzvLk+QRJdKSR7RdRtcj2i0teUSb5VHTiHYToAy8RbRb3qAS0Wbprke03fjPXa/8VlNJJ0h6UNLTkrZI+kqPMpJ0naStkp6UdFrVeiOGZdxfXVxHSz4FfNX245IOB34u6T7bT3eVORdYXmwfBb5b/IxovoO9Jbe9w/bjxf5vgGeA42YVWwXc5o5HgCMlLatad8SCc2d0fZCtqSoneTdJJwEfBh6ddeg44KWuz9v4/X8IIprJA24VSDpK0n2Sni9+Lpmj3LSkTcU2Oci5a0tySW8Hvg/8le1XDvAcayRtlLTxN3um6gotohLZA20VrQV+Yns58JPicy+v2f5QsZ0/yIlrSXJJi+kk+O22f9CjyHbghK7Pxxff/T+219mesD1x+FEZ+I+G2L86TNlWzSrg1mL/VuCCqifcr47RdQE3A8/Y/vYcxSaBS4pR9jOAvbZ3VK07YsEZmBlwg6X7e6LFtmYeNR3TlRO/BI6Zo9xbi3M/IumCQU5cR3P5ceBzwGZJm4rvvgG8G8D2jcAG4DxgK/Aq8Pka6o1YcGJeXfFdtifmPJd0P3Bsj0NXdn+wbWnOm3In2t4u6RTgAUmbbb/QL6jKSW77P+lM8e1XxsCXq9YVMRIz9azJbPvsuY5JelnSMts7ijtPO+c4x/bi54uSHqIz0N03yWsdXY9onfl116uYBFYX+6uBH84uIGmJpMOK/aV0etFPzy43W5I8osSQRtevBc6R9DxwdvEZSROSbirKvA/YKOkJ4EHg2lmTznrKEHZEmSHMeLO9Gzirx/cbgcuL/Z8BH5zvuZPkEX2N/wMqSfKIfrJaa0T7ZdGIiLZLkke0mIGZJHlEi2XgLaL9kuQRLWZgup5praOSJI/oy+AkeUS7pbse0WIZXY84CKQlj2i5JHlEi9kwPT3qKCpJkkeUSUse0XJJ8og2c0bXI1rN4EyGiWi5TGuNaDG7tiWZRyVJHlEmA28R7ea05BFtlkUjItotD6hEtJsBj/m01jpeXXyCpAclPS1pi6Sv9CizQtJeSZuK7aqq9UYMhYtFIwbZGqqOlnwK+KrtxyUdDvxc0n093tH0U9ufrqG+iKHywd5dL16cvqPY/42kZ4DjGOBtixFjocGt9CDkGkcOJZ0EPAx8wPYrXd+vAL4PbAN+AXzN9pYe//81wJri4weAp2oLrh5LgV2jDqJL4unvvbYPr3ICSf9O5881iF22V1apbyHUluSS3g78B/C3tn8w69g7gBnb+ySdB/yj7eUl59toe6KW4GrStJgST39Ni2dUank/uaTFdFrq22cnOIDtV2zvK/Y3AIuLl6hHxAKrY3RdwM3AM7a/PUeZY4tySDq9qHd31bojolwdo+sfBz4HbJa0qfjuG8C7AWzfCFwIfEnSFPAacJHLrxPW1RBb3ZoWU+Lpr2nxjEStA28R0Ty1XJNHRHMlySNarjFJLukoSfdJer74uWSOctNd02MnFyCOlZKelbRV0toexw+TdFdx/NFibsCCGiCmSyX9quv3cvkCxrJe0k5JPecwqOO6ItYnJZ22ULHMI6aDe1q17UZswLeAtcX+WuCbc5Tbt4AxLAJeAE4BDgWeAE6dVeYvgBuL/YuAuxb49zJITJcC1w/pv9MngNOAp+Y4fh7wY0DAGcCjDYhpBfBvw/j9NHFrTEsOrAJuLfZvBS4YQQynA1ttv2j7DeDOIq5u3XHeDZy1//bgCGMaGtsPA3v6FFkF3OaOR4AjJS0bcUwHtSYl+THuzIMH+CVwzBzl3ippo6RHJF1QcwzHAS91fd5WfNezjO0pYC9wdM1xzDcmgM8U3eO7JZ2wgPGUGTTeYTtT0hOSfizp/aMOZpiG+jy5pPuBY3scurL7g21Lmuve3om2t0s6BXhA0mbbL9Qd65j5EXCH7dclfYFOT+OTI46pSR6n8/dm/7Tqe4C+06rbZKhJbvvsuY5JelnSMts7iu7dzjnOsb34+aKkh4AP07lmrcN2oLsVPL74rleZbZIOAY5gYWfvlcZku7v+m+iMb4zKIL/DoXLXw1K2N0j6Z0lLbTfpYZoF06Tu+iSwuthfDfxwdgFJSyQdVuwvpTPbrs5HWh8Dlks6WdKhdAbWZo/gd8d5IfCAi9GdBVIa06xr3vOBZxYwnjKTwCXFKPsZwN6uy7CROOinVY965K9rBPRo4CfA88D9wFHF9xPATcX+x4DNdEaYNwOXLUAc5wHP0ekdXFl8dw1wfrH/VuBfga3AfwGnDOF3UxbT3wNbit/Lg8AfLWAsd9BZP+BNOtfblwFfBL5YHBdwQxHrZmBiCL+fspiu6Pr9PAJ8bFR/z0exZVprRMs1qbseEQsgSR7RcknyiJZLkke0XJI8ouWS5BEtlySPaLn/A/5p9xT7QxrZAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Show weights\n",
    "\n",
    "plt.imshow(nn.weights[1])\n",
    "plt.colorbar()\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ba66c1e7",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}