Tutorials/CNN/CNN3.0.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Includes Channel and Batch\n",
    "\n",
    "Coded by Lujia Zhong @lujiazho"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from miniml.ann import *\n",
    "from miniml.ann.common import *\n",
    "from miniml.ann.utils import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv('./datasets/digit-recognizer/train.csv')\n",
    "\n",
    "train_label = train['label']\n",
    "train = train.drop(['label'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAM90lEQVR4nO3df6zddX3H8derPyhJC7NX6KUrnSBrljUmFnNTnTWOSSRAshQTMVZD6kK8Rm1WnMsg7A/ZfwxBptvE1NFRjcKMQuiSRm0qGXEQwm3t2mIdsFq1P9I76B8U0fa2fe+P+2W5lns+53LO95zv6X0/H8nJOef7Pt/zfeekr37O+X7OuR9HhADMfnOabgBAfxB2IAnCDiRB2IEkCDuQxLx+HuwCL4gLtbCfhwRS+a1+rVNx0tPVugq77eslfVnSXEn/EhF3lx5/oRbq3b62m0MCKHgmdrSsdfw23vZcSf8s6QZJKyWts72y0+cD0FvdfGZfLenFiDgQEackPSJpbT1tAahbN2FfJulXU+4fqrb9Dtujtsdsj03oZBeHA9CNbsI+3UmAN3z3NiI2RcRIRIzM14IuDgegG92E/ZCk5VPuXy7pSHftAOiVbsL+rKQVtq+0fYGkj0raWk9bAOrW8dRbRJy2vUHSDzQ59bY5Ip6rrTMAtepqnj0itknaVlMvAHqIr8sCSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kERfl2wG+mnxfw61rD1y5Y+K+77z7z9TrF/25ac66qlJjOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kATz7DhvDT99cbH+1eWtFxieiPnFfR0dtTTQugq77YOSTkg6I+l0RIzU0RSA+tUxsv9ZRLxUw/MA6CE+swNJdBv2kPRD2zttj073ANujtsdsj03oZJeHA9Cpbt/Gr4mII7aXSNpu+2cR8eTUB0TEJkmbJOliD83C0x7A+aGrkT0ijlTX45Iek7S6jqYA1K/jsNteaPui129Luk7SvroaA1Cvbt7GD0t6zPbrz/PtiPh+LV0Bkg7c8yfF+iOX31esL/CClrX37FpX3Pf3HyqPW2eK1cHUcdgj4oCkd9bYC4AeYuoNSIKwA0kQdiAJwg4kQdiBJPiJKxpz/C/KU2tPr7u3WF8058Ji/Ysvr2xZG/5E+bdbZ155pVg/HzGyA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EASzLOjp+b+0R+2rK393BPFfX+vzTz6nlPlH5o+fu8HWtbe8vLTxX1nI0Z2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCeXZ0ZeK68sK9H7jvP1rW/mroZ10d+5P3bCzWL/1Gvrn0EkZ2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCeXYUHfvL9xbrO2//p2L9rKJl7fmJU8V9b/3pLcX60scOFOuni9V82o7stjfbHre9b8q2Idvbbb9QXS/ubZsAujWTt/EPSbr+nG13SNoRESsk7ajuAxhgbcMeEU9KOn7O5rWStlS3t0i6qd62ANSt0xN0wxFxVJKq6yWtHmh71PaY7bEJnezwcAC61fOz8RGxKSJGImJkvhb0+nAAWug07MdsL5Wk6nq8vpYA9EKnYd8qaX11e72kx+tpB0CvtJ1nt/2wpGskXWL7kKQvSLpb0nds3yrpl5Ju7mWT6J15V/xBsf7x0R/07Ng3j32yWF/+4X3FOvPob07bsEfEuhala2vuBUAP8XVZIAnCDiRB2IEkCDuQBGEHkuAnrrPc3OGW32SWJL3/3/cX67ctfr7NEVys/vz0b1vWFm67qM1zo06M7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBPPss93Fi4rlbpdNbue2d/15y9rQyyyp3E+M7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBPPss8C8y5e1rK3+bnkefU6b36O387mj7y7W4zetf8+O/mJkB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkmGefBca/trBl7c5L9hb3PdvmuTceWVOs//xPy+PF2ddea3ME9Evbkd32ZtvjtvdN2XaX7cO2d1eXG3vbJoBuzeRt/EOSrp9m+/0Rsaq6bKu3LQB1axv2iHhS0vE+9AKgh7o5QbfB9p7qbf7iVg+yPWp7zPbYhE52cTgA3eg07A9IukrSKklHJd3X6oERsSkiRiJiZL4WdHg4AN3qKOwRcSwizkTEWUlfl7S63rYA1K2jsNteOuXuhyTta/VYAIOh7Ty77YclXSPpEtuHJH1B0jW2V0kKSQclfap3LaL0e3VJ+uCyzv/2+6tny+dRdn7l6mL9La/xt9/PF23DHhHrptn8YA96AdBDfF0WSIKwA0kQdiAJwg4kQdiBJPiJ6wCY97blxfpF3/51sf53S37SsvbSmd8U973h3r8p1oe/+VSxjvMHIzuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJME8+wD4xbryPPtPrvjHjp/79sPlP/w7/BXm0bNgZAeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJJhn74Pxz7y3WH/0019s8wwXFqsbDr+vZe3ljw+1ee5X2tQxWzCyA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EASzLPXYO6llxbrf73x34r1K+eV59Hb2fXAqpa1oQMsqYxJbUd228ttP2F7v+3nbG+stg/Z3m77hep6ce/bBdCpmbyNPy3p8xHxx5LeI+mztldKukPSjohYIWlHdR/AgGob9og4GhG7qtsnJO2XtEzSWklbqodtkXRTj3oEUIM3dYLO9hWSrpb0jKThiDgqTf6HIGlJi31GbY/ZHpvQyS7bBdCpGYfd9iJJ35N0W0TM+NcTEbEpIkYiYmS+FnTSI4AazCjstudrMujfiohHq83HbC+t6ksljfemRQB1aDv1ZtuSHpS0PyK+NKW0VdJ6SXdX14/3pMPzwOGPrSjWP7Lo+z09/qmL3dPnx+wwk3n2NZJukbTX9u5q252aDPl3bN8q6ZeSbu5JhwBq0TbsEfFjSa2GjmvrbQdAr/B1WSAJwg4kQdiBJAg7kARhB5LgJ641mDNRrk/EmWJ9vucW6yejfIATV7V+/suKeyITRnYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIJ59hos+epTxfq/briqWF84p/znuu7/2oeL9RX/UD4+IDGyA2kQdiAJwg4kQdiBJAg7kARhB5Ig7EASzLP3wdaVb+1q/8vEPDq6x8gOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0m0Dbvt5bafsL3f9nO2N1bb77J92Pbu6nJj79sF0KmZfKnmtKTPR8Qu2xdJ2ml7e1W7PyLu7V17AOoyk/XZj0o6Wt0+YXu/pGW9bgxAvd7UZ3bbV0i6WtIz1aYNtvfY3mx7cYt9Rm2P2R6bUPnPLwHonRmH3fYiSd+TdFtEvCLpAUlXSVqlyZH/vun2i4hNETESESPztaD7jgF0ZEZhtz1fk0H/VkQ8KkkRcSwizkTEWUlfl7S6d20C6NZMzsZb0oOS9kfEl6ZsXzrlYR+StK/+9gDUZSZn49dIukXSXtu7q213Slpne5WkkHRQ0qd60B+AmszkbPyPJXma0rb62wHQK3yDDkiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kIQjon8Hs/9X0i+mbLpE0kt9a+DNGdTeBrUvid46VWdvb4uIS6cr9DXsbzi4PRYRI401UDCovQ1qXxK9dapfvfE2HkiCsANJNB32TQ0fv2RQexvUviR661Rfemv0MzuA/ml6ZAfQJ4QdSKKRsNu+3vZ/237R9h1N9NCK7YO291bLUI813Mtm2+O2903ZNmR7u+0Xqutp19hrqLeBWMa7sMx4o69d08uf9/0zu+25kp6X9EFJhyQ9K2ldRPy0r420YPugpJGIaPwLGLbfL+lVSd+IiHdU2+6RdDwi7q7+o1wcEbcPSG93SXq16WW8q9WKlk5dZlzSTZI+oQZfu0JfH1EfXrcmRvbVkl6MiAMRcUrSI5LWNtDHwIuIJyUdP2fzWklbqttbNPmPpe9a9DYQIuJoROyqbp+Q9Poy442+doW++qKJsC+T9Ksp9w9psNZ7D0k/tL3T9mjTzUxjOCKOSpP/eCQtabifc7VdxrufzllmfGBeu06WP+9WE2GfbimpQZr/WxMR75J0g6TPVm9XMTMzWsa7X6ZZZnwgdLr8ebeaCPshScun3L9c0pEG+phWRByprsclPabBW4r62Osr6FbX4w338/8GaRnv6ZYZ1wC8dk0uf95E2J+VtML2lbYvkPRRSVsb6OMNbC+sTpzI9kJJ12nwlqLeKml9dXu9pMcb7OV3DMoy3q2WGVfDr13jy59HRN8vkm7U5Bn5/5H0t0300KKvt0v6r+ryXNO9SXpYk2/rJjT5juhWSW+VtEPSC9X10AD19k1JeyXt0WSwljbU2/s0+dFwj6Td1eXGpl+7Ql99ed34uiyQBN+gA5Ig7EAShB1IgrADSRB2IAnCDiRB2IEk/g+/DMuBLxBsJgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "img = np.array(train.iloc[0]).reshape(28, 28)\n",
    "_ = plt.imshow(img)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Conv2D(Node):\n",
    "    def __init__(self, nodes, weights, bias, output_c, input_c, stride=1, padding=False):\n",
    "        self.output_c = output_c\n",
    "        self.input_c = input_c\n",
    "        self.stride = stride\n",
    "        self.padding = padding\n",
    "        self.bias = True if bias else False\n",
    "        Node.__init__(self, [nodes, weights, bias] if bias else [nodes, weights])\n",
    "\n",
    "    def conv_(self, n_c, n_h, n_w, img, filter_, s=1):\n",
    "        k = filter_.shape[2]\n",
    "        res = np.zeros((self.inputs[0].value.shape[0], n_c, n_h, n_w))\n",
    "        aug = np.repeat(img, filter_.shape[0], axis=0)\n",
    "        aug = aug.reshape(img.shape[0], filter_.shape[0], img.shape[1], img.shape[2], img.shape[3])\n",
    "        for i in range(n_h):\n",
    "            for j in range(n_w):\n",
    "                res[:,:,i,j] = np.sum(aug[:,:,:,i*s:i*s+k,j*s:j*s+k]*filter_, axis=(2,3,4))\n",
    "        return res\n",
    "        \n",
    "    def forward(self):\n",
    "        # inputs: [batch, channel_in, hight_in, width_in]\n",
    "        inputs = self.inputs[0].value\n",
    "        # weights: [channel_out, channel_in, kernel_size, kernel_size]\n",
    "        weights = self.inputs[1].value\n",
    "        # bias: [channel_out, 1, 1] or [1, channel_out, 1, 1] is the same\n",
    "        if self.bias:\n",
    "            bias = self.inputs[2].value\n",
    "        k = weights.shape[2]  # kernel size (w equals to h)\n",
    "\n",
    "        s = self.stride\n",
    "        if self.padding:\n",
    "            inputs = np.pad(inputs, ((0,0),(0,0),(k//2,k//2),(k//2,k//2)), 'constant')\n",
    "        assert (inputs.shape[2]-k)%s == 0\n",
    "        assert (inputs.shape[3]-k)%s == 0\n",
    "        assert s < k\n",
    "\n",
    "        n_c = self.output_c\n",
    "        new_h = (inputs.shape[2]-k)//s+1\n",
    "        new_w = (inputs.shape[3]-k)//s+1\n",
    "\n",
    "        # value: [batch, channel_out, hight_out, width_out]\n",
    "        self.value = self.conv_(n_c, new_h, new_w, inputs, weights, s=s) + (bias if self.bias else 0)\n",
    "    \n",
    "    def backward(self):\n",
    "        # create zero loss for all inputs\n",
    "        self.gradients = {n: np.zeros_like(n.value, dtype=np.float64) for n in self.inputs}\n",
    "        s = self.stride\n",
    "        k = self.inputs[1].value.shape[2]\n",
    "        X = self.inputs[0].value\n",
    "\n",
    "        for n in self.outputs:\n",
    "            # Get the cost w.r.t this node.\n",
    "            grad_y = n.gradients[self]\n",
    "            \n",
    "            # calculate inserted gradient_y\n",
    "            if s > 1:\n",
    "                tmp = np.insert(grad_y.reshape(grad_y.shape[0],grad_y.shape[1],-1,1), 1, [[[[0]]]]*(s-1), axis=3)\n",
    "                tmp = tmp.reshape(grad_y.shape[0],grad_y.shape[1],grad_y.shape[2],-1)[:,:,:,:-(s-1)]\n",
    "                tmp = np.insert(tmp, np.repeat(range(1,grad_y.shape[2]), (s-1)), [0]*tmp.shape[3], axis=2)\n",
    "                inser_grad = np.pad(tmp, ((0,0),(0,0),(k-1,k-1),(k-1,k-1)), 'constant')\n",
    "            else:\n",
    "                inser_grad = np.pad(grad_y, ((0,0),(0,0),(k-1,k-1),(k-1,k-1)), 'constant')\n",
    "            \n",
    "            # W: [ch_out, ch_in, kernel_size, kernel_size], rotate 2 times in h and w dimension\n",
    "            rotated_w = np.rot90(self.inputs[1].value, 2, axes=(2,3))\n",
    "            \n",
    "            # X: [batch, ch_in, height_in, width_in]\n",
    "            n_h = self.gradients[self.inputs[0]].shape[2]\n",
    "            n_w = self.gradients[self.inputs[0]].shape[3]\n",
    "\n",
    "            # A special but useful trick to calculate all together\n",
    "            aug1 = np.repeat(inser_grad, rotated_w.shape[1], axis=1)\n",
    "            aug1 = aug1.reshape(inser_grad.shape[0], inser_grad.shape[1], \n",
    "                                rotated_w.shape[1], inser_grad.shape[2], inser_grad.shape[3])\n",
    "            aug2 = np.repeat(rotated_w, aug1.shape[1], axis=0)\n",
    "            aug2 = aug2.reshape(rotated_w.shape[0], aug1.shape[1], \n",
    "                                rotated_w.shape[1], rotated_w.shape[2], rotated_w.shape[3])\n",
    "            re_aug1 = np.repeat(aug1, aug2.shape[0], axis=0)\n",
    "            re_aug1 = re_aug1.reshape(aug1.shape[0], aug2.shape[0], \n",
    "                                aug1.shape[1], aug1.shape[2], aug1.shape[3], aug1.shape[4])\n",
    "\n",
    "            grad_X = np.zeros_like(X, dtype=np.float64)\n",
    "            shift = k//2 if self.padding else 0\n",
    "            for i in range(n_h):\n",
    "                for j in range(n_w):\n",
    "                    # dim 0 for batch, dim 1 and 2 for ch_out, dim 3 for ch_in, dim 4 and 5 for h&w\n",
    "                    grad_X[:,:,i,j] = np.sum(re_aug1[:,:,:,:,i+shift:i+shift+k,j+shift:j+shift+k]*aug2, axis=(1,2,4,5))\n",
    "            self.gradients[self.inputs[0]] += grad_X\n",
    "            \n",
    "            # W: [ch_out, ch_in, kernel_size, kernel_size]\n",
    "            aug1 = np.repeat(inser_grad, X.shape[1], axis=1)\n",
    "            aug1 = aug1.reshape(inser_grad.shape[0], inser_grad.shape[1], X.shape[1],\n",
    "                                inser_grad.shape[2], inser_grad.shape[3])\n",
    "            aug2 = np.repeat(X, aug1.shape[1], axis=0)\n",
    "            aug2 = aug2.reshape(X.shape[0], aug1.shape[1], X.shape[1], X.shape[2], X.shape[3])\n",
    "            re_aug1 = np.repeat(aug1, aug2.shape[0], axis=0)\n",
    "            re_aug1 = re_aug1.reshape(aug1.shape[0], aug2.shape[0], \n",
    "                                aug1.shape[1], aug1.shape[2], aug1.shape[3], aug1.shape[4])\n",
    "            \n",
    "            grad_W = np.zeros_like(self.inputs[1].value, dtype=np.float64)\n",
    "            \n",
    "            for i in range(grad_W.shape[2]):\n",
    "                for j in range(grad_W.shape[3]):\n",
    "                    # dim 0 and 1 are for batch, 4 and 5 are for h and w, 2 and 3 are for ch_out and ch_in\n",
    "                    grad_W[:,:,i,j] = np.sum(re_aug1[:,:,:,:,i:i+X.shape[2],j:j+X.shape[3]]*aug2, axis=(0,1,4,5))\n",
    "            self.gradients[self.inputs[1]] += np.rot90(grad_W, 2, axes=(2,3))\n",
    "            \n",
    "            if self.bias:\n",
    "                self.gradients[self.inputs[2]] += np.sum(grad_y, \n",
    "                                                         axis=(0,2,3), \n",
    "                                                         keepdims=False).reshape(grad_y.shape[1],1,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class AvePooling(Node):\n",
    "    def __init__(self, nodes, k=3):\n",
    "        self.k = k\n",
    "        Node.__init__(self, [nodes])\n",
    "\n",
    "    def conv_(self, n_c, n_h, n_w, img, filter_, s=1):\n",
    "        k = self.k\n",
    "        res = np.zeros((self.inputs[0].value.shape[0], n_c, n_h, n_w))\n",
    "        for i in range(n_h):\n",
    "            for j in range(n_w):\n",
    "                res[:,:,i,j] = np.sum(img[:,:,i*s:i*s+k,j*s:j*s+k]*filter_, axis=(2,3))\n",
    "        return res\n",
    "        \n",
    "    def forward(self):\n",
    "        inputs = self.inputs[0].value\n",
    "        k = self.k\n",
    "        \n",
    "        assert inputs.shape[2]%k == 0\n",
    "        assert inputs.shape[3]%k == 0\n",
    "        assert k > 1\n",
    "\n",
    "        filter_ = np.ones((k, k))/(k*k)\n",
    "        new_h = inputs.shape[2]//k\n",
    "        new_w = inputs.shape[3]//k\n",
    "        \n",
    "        self.value = self.conv_(inputs.shape[1], new_h, new_w, inputs, filter_, s=k)\n",
    "    \n",
    "    def backward(self):\n",
    "        # create zero loss for all inputs\n",
    "        self.gradients = {n: np.zeros_like(n.value, dtype=np.float64) for n in self.inputs}\n",
    "        s = k = self.k\n",
    "        X = self.inputs[0].value\n",
    "\n",
    "        for n in self.outputs:\n",
    "            # Get the cost w.r.t this node.\n",
    "            grad_y = n.gradients[self]\n",
    "\n",
    "            tmp = np.insert(grad_y.reshape(grad_y.shape[0],grad_y.shape[1],-1,1), 1, [[[[0]]]]*(s-1), axis=3)\n",
    "            tmp = tmp.reshape(grad_y.shape[0],grad_y.shape[1],grad_y.shape[2],-1)[:,:,:,:-(s-1)]\n",
    "            tmp = np.insert(tmp, np.repeat(range(1,grad_y.shape[2]), (s-1)), [0]*tmp.shape[3], axis=2)\n",
    "            inser_grad = np.pad(tmp, ((0,0),(0,0),(k-1,k-1),(k-1,k-1)), 'constant')\n",
    "            \n",
    "            w = np.ones((k, k))/(k*k)\n",
    "            \n",
    "            n_h = self.gradients[self.inputs[0]].shape[2]\n",
    "            n_w = self.gradients[self.inputs[0]].shape[3]\n",
    "            self.gradients[self.inputs[0]] += self.conv_(X.shape[1], n_h, n_w, inser_grad, w, s=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Flatten(Node):\n",
    "    def __init__(self, nodes):\n",
    "        Node.__init__(self, [nodes])\n",
    "        \n",
    "    def forward(self):\n",
    "        inputs = self.inputs[0].value\n",
    "        self.value = inputs.flatten().reshape(inputs.shape[0],-1)\n",
    "    \n",
    "    def backward(self):\n",
    "        # create zero loss for all inputs\n",
    "        self.gradients = {n: np.zeros_like(n.value, dtype=np.float64) for n in self.inputs}\n",
    "        X = self.inputs[0].value\n",
    "\n",
    "        for n in self.outputs:\n",
    "            # Get the cost w.r.t this node.\n",
    "            grad_cost = n.gradients[self]\n",
    "            self.gradients[self.inputs[0]] += grad_cost.reshape(X.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "W1 = np.random.randn(3, 1, 5, 5)\n",
    "b1 = np.random.randn(3, 1, 1)\n",
    "\n",
    "W2 = np.random.randn(10, 3, 3, 3)\n",
    "b2 = np.random.randn(10, 1, 1)\n",
    "\n",
    "W3 = np.random.randn(250, 100)\n",
    "b3 = np.random.randn(100)\n",
    "\n",
    "W4 = np.random.randn(100, 10)\n",
    "\n",
    "x_node, y_node = Placeholder(), Placeholder()\n",
    "\n",
    "W1_node = Placeholder()\n",
    "b1_node = Placeholder()\n",
    "\n",
    "W2_node = Placeholder()\n",
    "b2_node = Placeholder()\n",
    "\n",
    "W3_node = Placeholder()\n",
    "b3_node = Placeholder()\n",
    "\n",
    "W4_node = Placeholder()\n",
    "\n",
    "conv1 = Conv2D(x_node, W1_node, b1_node, output_c=3, input_c=1, stride=1, padding=False)\n",
    "ave1 = AvePooling(conv1, k=2)\n",
    "conv2 = Conv2D(ave1, W2_node, b2_node, output_c=10, input_c=3, stride=1, padding=False)\n",
    "ave2 = AvePooling(conv2, k=2)\n",
    "flat = Flatten(ave2)\n",
    "l1 = Linear(flat, W3_node, b3_node)\n",
    "tanh1 = Sigmoid(l1)\n",
    "l2 = Linear(tanh1, W4_node, None)\n",
    "tanh2 = Sigmoid(l2)\n",
    "mse = MSE(y_node, tanh2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "feed_dict = {\n",
    "    x_node: np.array([img]),\n",
    "    y_node: np.array([0,1,0,0,0,0,0,0,0,0]),\n",
    "\n",
    "    W1_node: W1,\n",
    "    b1_node: b1,\n",
    "\n",
    "    W2_node: W2,\n",
    "    b2_node: b2,\n",
    "    \n",
    "    W3_node: W3,\n",
    "    b3_node: b3,\n",
    "    \n",
    "    W4_node: W4\n",
    "}\n",
    "\n",
    "graph = feed_dict_2_graph(feed_dict)    # network graph\n",
    "sorted_graph = topology(graph)          # sorted graph\n",
    "trainables = [W1_node, b1_node, W2_node, b2_node, W3_node, b3_node, W4_node]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1, Loss: 0.1056\n",
      "Epoch: 2, Loss: 0.0955\n",
      "Epoch: 3, Loss: 0.0890\n",
      "Epoch: 4, Loss: 0.0871\n",
      "Epoch: 5, Loss: 0.0834\n",
      "Epoch: 6, Loss: 0.0812\n",
      "Epoch: 7, Loss: 0.0792\n",
      "Epoch: 8, Loss: 0.0792\n",
      "Epoch: 9, Loss: 0.0741\n",
      "Epoch: 10, Loss: 0.0741\n",
      "Epoch: 11, Loss: 0.0725\n",
      "Epoch: 12, Loss: 0.0694\n",
      "Epoch: 13, Loss: 0.0701\n",
      "Epoch: 14, Loss: 0.0684\n",
      "Epoch: 15, Loss: 0.0658\n",
      "Epoch: 16, Loss: 0.0618\n",
      "Epoch: 17, Loss: 0.0609\n",
      "Epoch: 18, Loss: 0.0611\n",
      "Epoch: 19, Loss: 0.0567\n",
      "Epoch: 20, Loss: 0.0557\n",
      "23.1986 sec/epoch\n"
     ]
    }
   ],
   "source": [
    "start_time = time.time()\n",
    "optimizer = 'Adam'\n",
    "lr = 1e-2\n",
    "epochs = 20\n",
    "batch = 2\n",
    "\n",
    "# only used for Adam\n",
    "it = 0\n",
    "\n",
    "for i in range(1, epochs+1):\n",
    "    it += 1\n",
    "    losses = []\n",
    "    for j in range(1, 1000, batch):\n",
    "        # Step 4.1: sample a batch of examples and Reset value\n",
    "        x_node.value = np.array(train.iloc[j:j+batch]).reshape(batch, 1, 28, 28)\n",
    "        y_node.value = np.eye(10)[train_label[j:j+batch]].reshape(batch,-1)\n",
    "\n",
    "        # Step 4.2: forward\n",
    "        for n in sorted_graph:\n",
    "            n.forward()\n",
    "\n",
    "        # Step 4.3: backward\n",
    "        for n in sorted_graph[::-1]:\n",
    "            n.backward()\n",
    "\n",
    "        # Step 4.4: optimization\n",
    "        for t in trainables:\n",
    "            t.optimize(optimizer=optimizer, lr=lr, it=it)\n",
    "\n",
    "        # Step 5: update current loss\n",
    "        loss = sorted_graph[-1].value\n",
    "        \n",
    "        losses.append(loss)\n",
    "\n",
    "    if (i+1) % 1 == 0:\n",
    "        print(\"Epoch: {}, Loss: {:.4f}\".format(i, np.mean(losses)))\n",
    "print(f\"{((time.time() - start_time)/epochs):.4f} sec/epoch\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.632\n"
     ]
    }
   ],
   "source": [
    "preds = []\n",
    "reals = []\n",
    "for i in range(1):\n",
    "    x_node.value = np.array(train.iloc[i:i+1000]).reshape(1000, 1, 28, 28)\n",
    "    for n in sorted_graph[:-1]:\n",
    "        n.forward()\n",
    "    preds.append(np.argmax(sorted_graph[-2].value, axis=1))\n",
    "    reals.append(train_label[i:i+1000])\n",
    "print(np.sum(np.array(preds)==np.array(reals))/1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAC4CAYAAAD61bdSAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAdOUlEQVR4nO2daYxc53Wm31NL73uzNzZXcZNkSaRiSvKimdiOnTiaAHKQxLAzM5AxDpgANhBj8keY/HAQzI/8GDszwAxiKGOPlIljZ1XkxMYktCZjRbFsi1ZoiSJFcd+62d0ku7t6qaqu5ZsfLA044nuK7O7qqv7I9wEa3X1ufcu9de6pW/d77zkWQoAQQoj4SDR6AkIIIVaGArgQQkSKArgQQkSKArgQQkSKArgQQkSKArgQQkTKqgK4mX3czI6b2Ukze7pWkxKi0ci3RQzYSnXgZpYE8DaAjwG4COBVAJ8OIRz12qS62kJ6sGdF4603ykXns8+qNCo6G9NlZxC/s2S6xJuU/c/kRIKPE0K1ScdDYXIGxcziqndmJb7d1N0aWoa7Vjv0uiDl+Emxim+VAt9W9M6TUrVrRycmVXlnEyk+54TxvhIJP+6ZN34DyV3OYGk2e9MRSK2iz0cBnAwhnAYAM/smgCcBuE6eHuzBzi//2iqGXBtKVRzT+3xbvNrGN6T8Nz91JU3t5Y05bl/w357ekQyfV67JbdPemqf2QilJ7bGF9ZP//r/Xqqtl+3bLcBce+cq/rtX4NSNRJRiZE9x6mrPUPpNvdfuayfFtU1c7qT3M+n4aks6cPTuAzsF5am9O8QudrhZ+zgF+0G8kr/7G16l9NbdQRgFcuOH/ixWbELEj3xZRsJoAzi7QbvroMrMDZnbIzA6VMourGE6IurFs316a5VetQqwlqwngFwFsvuH/TQDG3v2iEMIzIYT9IYT9yS7ntoMQ64tl+3ZTt397QYi1YjX3wF8FsMvMtgO4BOBTAH61JrNaBeUqC3KLC83Ubpe5HQASBd5fW4bbW69UuX/mbGp9hd8PnNnF700DwNxsH7X33H/VH99hoGOB2qfm25fdV2z3zR3WpW9Xu5/d2cTv6Q63zLltTmQGqP1Hp7ZRu13x71t3nuHXgr1ZPmdz1u2rUuXUWhjtpfb5IX4PPDPof2Ma6Ob301vTBX8CDWLFATyEUDSzzwP4OwBJAF8LIbxZs5kJ0SDk2yIWVnMFjhDCdwB8p0ZzEWLdIN8WMaAnMYUQIlIUwIUQIlIUwIUQIlIUwIUQIlJWtYjZSLzH3xcu+9K3RJ63aZnyP8cGDi/x8Vt4m/Qcly0BwJWHHLmik0ei5Ku20HWS2xcyG9w25Ye4pKyjmT9iv2/oktvXkakRaq+WlkDcHp5ccFunLxGdK7RQ+8HTe9w2xTH+XMbwD/nrOy74j5+nL/K5hXZHH18oun3ltnJJYLHdl9W2X+bHbH4TD3GZ7X6cuDTIT7yhjTPUXk1e6OWVqRU624QQIlIUwIUQIlIUwIUQIlIUwIUQIlIUwIUQIlKiVaHkc7w4ghX8z6SO83xb1zlfOZIo8NXtmZ380JVa/EPqiE1Q6OIpoHLD/rwKHV5FID/jT/IYT64/9SBfKc8W+DEGgPeNnKP2I9e4OmU2y1US4mZaU1zVsFT2fevNqWFqTx7pcNsMH+ZKkLaDr1N7OeerUMIAT4xlxn07tPn+kFrk80rm/POh2MF9teMi7ys97ytaZndwFcpEkSeQ6xr2E4YNdvLEWLVCV+BCCBEpCuBCCBEpCuBCCBEpCuBCCBEpCuBCCBEp616Fks3yFeHivKOQSPu5B7rOcYVGMlclX4FTIyw7xPvKD/or5W0DvHRZ/ngXH9op5wYAzdN8W77fV6Es9fO55ad4ToymET9fxfHZQWp/ZANXpxzLcJUEAFyY6aH2RBVFzZ1Ae4rn2elI89w0h8Y3UzsALJznPjTylu+P7d/nCXXCzm3Unkj6/riwmSuc8t38GjE76F87Nk/z9z1RrKKw4ocS6QW+/y3T/nExJ8+QFy4zCV/pk0rycfraalMEW1fgQggRKQrgQggRKQrgQggRKQrgQggRKQrgQggRKQrgQggRKauSEZrZWQBzAEoAiiGE/bWY1I0UZnkZMlvikqaOc36SmpYpLt25stcp+wQg38PtniQv2eXomQDs2nCF27cfpfaZApf3AcCPvrmX2gde86VWC0P82Mxv58cyM+8flw+PnqD28Vw3tf9U7wW3r8UCl4pOL/rjrzX18O3+Zi4rnchxSd78lF8GrH2MX4u1XfblauVtPPHY9P1cklj03RH5Hu5D2WEu0W3a5CeAam3nc56Y4L4FAO3HnTjhlPjrOlNF7jvBz+HsBp6AK9/vx5zMHD9otZIR1kIH/uEQAo9MQsSNfFusa3QLRQghImW1ATwA+Hsz+7GZHWAvMLMDZnbIzA6VMourHE6IurEs316arc1XYiGWw2pvoXwwhDBmZoMADprZWyGEl258QQjhGQDPAEDrzo139rPR4k5iWb7dtcfJrSDEGrKqK/AQwljl9ySA5wE8WotJCdFo5NsiBlZ8BW5m7QASIYS5yt8/C+B3azazCslOXl4qecYpyVTlOmhhlK9Uo0ouq473T1H77o4MtV/J+kqBvT0XqX1L01VqfwtcJQAAnT93mY//oyG3zaZ/4CWxek/yVfSJ/b7s4NQoL6H1hU0Hqf178/e6fW3pvEbt1xZH3TZ+WqXVUy/fvprnvpLJO75dJVFb7wmuqrCS38ZTm2TucRRej/jruds6uKrkpze8Te0PtvBzAQAuFXqp/fxov9vmj1P887XpBFcy5Xr9a1cr80R5zTP8WCazvjcWl/i5NZP1FVY9rbd/O241t1CGADxfqXmXAvAnIYT/tYr+hFgvyLdFFKw4gIcQTgPgYmQhIka+LWJBMkIhhIgUBXAhhIgUBXAhhIgUBXAhhIiUdVETM5d36lsCSCS4dMdKXLrTesXXERZaeZvZx7i8DgB+fcthan+49Sy1/+cLH3P76kjycc4vcXnUt0+/x+2rXF6+kC6zlcsoO8/z5D2bD/JkSwDwk9Et1D6zkUsPd7ZMuH0VApdanW7Z4LaZyzmS0HVGc9KvK1oMTr3IAj8fUpM86RcANM3y9zDf7x+nq3v5udJ3H5cLfnHP37h9PdjE24wmuT9cKvlPZR/Ncfno7hYunQWAA/tepvavLH6I2hNFP+YUOrg/dow5MsK8fy4WcryvzIIjFcXyZIS6AhdCiEhRABdCiEhRABdCiEhRABdCiEhRABdCiEhZFyqUcsn/HOlo58qN5AW+ut37Bk8yBQDz93RQ+4f38IQ7ADCSnqb2uTJPRnPs9Ea3r/EMTx70qztepfbdAzyRFgDM5Pn45zZWUWec5ivv2UFu7/0RT7IFABsP8qRZ397Ln0D/zMA/un3NlfiK/GCbX3YrFhVKyvxkUi0pnqht0rifNs34aoeFYf4eFlv8Np7a5N9t/77bxuNskc/5conv44WCf57knGRS/2fGT4jWk+aqlp07uXLlTGaT21frJD9mZadyWqqaaMR5+xOJ2mQf1hW4EEJEigK4EEJEigK4EEJEigK4EEJEigK4EEJEyrpQobS25d1tO/r4Svl4pofarcBLS13fyM3b2/xSUV1O/hKv7BOqrC7PTvMSWv/jrfdTeyrl78sn7/lnav8u/JX6sT28RNvmg1wpUBjpcfvq/udJaj86zdUpfUN+vpl9Leep/eXULrdNLMwVfLXMlnZeSu7tEi9X1zzj+1apmTt3vt9XobQb7++aoyg5lRt0+5rId1L7YpHnbykHf177e7g/JKooehZK/Dg/1n+W2k/2+qUH7bIz5xSfczpTRVGS5ttKTi4nACiWb76uDk7w0hW4EEJEigK4EEJEigK4EEJEigK4EEJEigK4EEJEigK4EEJEyi1lhGb2NQC/AGAyhPBAxdYH4E8BbANwFsAnQwg869O7YPKh+SkurwMA9POETp2neKKjkHYyzgBoneBlpx5uO+u28UpFvTK/k9q7ev1SUSUiDwKApFM2rhoXclzG+MiGc26b1x5xxjnIZWvJBX68ACCM8RJp117Zx+27/RJSw0l+zLa1+cm0jiZuloGZI4vzqLVvJ3Dz+Fs6/KaTjvRuepwnPeuqkpiq/wjPqDTVzpOeAcCVaT7+txO8lF9H2vcHj7Kn3a3CuVwftQ83+4nqTi1wH25PcolyosWX6BZ5njw4OddcefL1bdwngxMLACAwiaXj2rdzBf4sgI+/y/Y0gBdDCLsAvFj5X4jYeBbybRExtwzgIYSXALz7iYMnATxX+fs5AJ+o7bSEWHvk2yJ2VnoPfCiEMA4Ald/uI1pmdsDMDpnZoVLGv70gxDphRb5dmL39SuJC1Io1X8QMITwTQtgfQtif7HJuLgkRITf6drrbv9csxFqx0gA+YWYjAFD5zRNjCBEf8m0RDStNZvUtAE8B+L3K7xdWM4lke9Hd9tqprdS+521eBi0x0O/2Vejly8hHc6Numw8084RDm5q4uqA57e9LvlC73GE/GNtG7V7yLwD4za3fpfYv3v8Zah/6oa86SA7xVf+Bw3z/S0/51wptzip+2qokJls7aurbrUmeKAwACoEfk9Qs95OEn/MNqWl+C8dK/jeDcoGPn8ny8yRdRS2VTtbuvTo9t4Ha5wq+kmm0dYbaZwr8W39P94Lb12Jyed+mnHx31yk6yrMqieqWwy2vwM3sGwBeAbDHzC6a2Wdx3bk/ZmYnAHys8r8QUSHfFrFzy0vCEMKnnU0/U+O5CFFX5NsidvQkphBCRIoCuBBCRIoCuBBCREpdS6qZBbpavX/HSbfNK3/3IN+wlStHqqQYQDnN5Q4DKZ5XBQDSxjucdVb3S+Xl536oJWPz3e62svN5ndnJ1QUjL/rL62GRqx7azvur+x4jKV7Cqy/l97XcvCdrTYBhqXxzHp7jGb8M2X3dl6k9Nc99KJX19zk3yo9hetFvE7I8b1DoaqwPe8ws+eqQh7t5GTZPhdLV4kt6so5wKL3g5DVJVDleKX5updO+CoX6tjOErsCFECJSFMCFECJSFMCFECJSFMCFECJSFMCFECJSFMCFECJS6iojBLhEZrRlxn19wpH0FAa4PCiR8+U5rad4ia6BlF+q6VyRS4f2t52h9hfsIbevejCfa3a3feXCT1P74B5etg6TfjmwpYe2UXv60Alq7zJftlUKad6X+YnBmHJrPYrf8kX/FCsGLuPL93PpWSrnX28VW5uovfVqlQRUs8743fz9uJTrcfvaNMB9JbWCcoEeXklCAOhLcslptsT35eIUL0kIAJ1X+TnfeYH78OQ+X96YauM+PNo967ZhJdXMqammK3AhhIgUBXAhhIgUBXAhhIgUBXAhhIgUBXAhhIiU+iazAlcPHOh7xW3zQu5fUHsyy1d3cwN+2SV7hStHPv+9f+O2+Y+PP0/tr2R2UvtjQ+fcvr4/vo3ai87qejVVRbHE2yxe8QtHn3XKO+Wv8VX0vvS821ehnbtO+f17qP3Bpu+5fXmcyA6529j+hwbqUAwBzcmbfXJ3r19S83Kui9qHdnNV0ITxUmMA0DbGFSXlJv8arcWZWiHLfajY7ifGOuf41oZ+niiuu8VPlJY0rlwZbfOVGy/P7qL2s5k+ak+c9pUjG95wytMVuMItO+wflw29fP8f7Blz27wxs9Hd9m50BS6EEJGiAC6EEJGiAC6EEJGiAC6EEJGiAC6EEJFyywBuZl8zs0kzO3KD7XfM7JKZHa78PLG20xSi9si3RezcjozwWQD/FcAfvcv++yGE/1SLScyRWoLvED7ApUOlH/OkTV7dSwCA8W0dx3kiIAD40sBHqb23jUuNclWSFz0wME7thyd4fc9cnifiAYDiGJd6WbJKrci3eN3E1jw/LsVtvowv18ffs8x2vv/ZsOT2VXIS9Ryf88evEc9ijX27P+3X9XxzZoTat3ddo/arg+1uX3aRv7fZQd8fuk5ze6uT2yzf659biyV+Pk6VeJsw4Pc10smTyx2f8euLXjrDJZbm1Kjd/dd+HVwrchljdpQf//JWHgsA4L0bLlJ72vykeyyZlcctr8BDCC8B4B4lRMTIt0XsrOYe+OfN7PXK11A/N6MQ8SHfFlGw0gD+BwB2ANgHYBzAl7wXmtkBMztkZoeKs4srHE6IurEi3y7M+l+jhVgrVhTAQwgTIYRSCKEM4A8BPFrltc+EEPaHEPanuv3HvIVYD6zUt9Pd/qPZQqwVKwrgZnbj6ssvAjjivVaImJBvi5i4pQrFzL4B4EMANpjZRQBfBPAhM9sHIAA4C+DXb2ewcjAsFW9WLzyfedht80s7DlP7nz/Oy4P1HfVXd5M7tlF7z0m/zaVRfgt0urOT2lu7/CQ9/3jmXmo3Z6W+edJX56Saubqg/YK/gu2Vp7PA+5p4lCsbACDPcwRh4+N81X2i5JdH+0FuK7WPz/FjDAAlslLv7IZLLX37+pxuvh6ad9QZANDbzG8p5kr8tLx3xE+M9eYI/3abzvjXaEvd3N7ilBRrnq52gB0l0yzf/4WT/nE5k+eKkqY5f/zhDN/WNsmdPnHGTyY1+xGeGCu7gR/LHUNcXQYAA01c7XI22++2KS8jKdstA3gI4dPE/NXbHkGIdYp8W8SOnsQUQohIUQAXQohIUQAXQohIUQAXQohIqWtJNY/XMzwXCAB8pO8tai8/yFd3Z/O+ciGV5avbHad47gUAGGzjS/VT7+WHLpf086o0T/I2Xu6JxJK/6u7lpSBCiP/H7B7en1PBCim/ohry27na5hMjP6H25ioL69+dvp+PUfBzwSQTzqTXGRNV/HFfN1fsvHxlB7UPtvr5O/a+9xS1/+THvC8ASCxxZyk6kva2Sf+Yp5x8Ok0ZrvBK5nzlV3qG+1ZwchkBgJWc/k5doObs4/e5fS2McPXX7ANc0fLLAyfcvmZL/GBezft5bZaDrsCFECJSFMCFECJSFMCFECJSFMCFECJSFMCFECJSFMCFECJS1oWM8MyMn9gl18ulZPcO8cQ+R/b6uzQ320Lt5bQv9ep+m5fEKid5oqerH60i/RvlZcWWerlsKT3nf76WU3yc7JA/frmDS63MkZNtf+SS21dmiScj+uXON6n9L+be4/b11jQvlZWIRCpYjcms71s72q5Q++4u7tsXF3vcvra0T1P73EP+e3jmtU3Unp535HpVpKDtF3k+9PT4DG+w5GRWA4Amfs6HFl+iazl+bmXfzxPITTzq97XUxc+hzzz2T9S+tZm/jwDw0uxuPq+iL5FdDroCF0KISFEAF0KISFEAF0KISFEAF0KISFEAF0KISFkXKhRWZu0dvn35QWp/rP8s76vs93XssY3UHl71V4SzfVxt0n0mT+0Lx/zitmVnmKVurrYobvHLs/2r+3ipxrczXNEBAFMLPIHOjt6r1N6Z4vsIAJ09fG4vzO+h9n+a8ZMq5Zb4gbn9wlLrF1Zm7R3eyHB/fE8XL9FVbPV9++TcALX3t3AVFQBc3XON2qe7uqh9ccQPF92neEm3ps38fOg65WdKK3RxhVN2wD9PZ+/hxzl7L/fTLSN+SbWPDvMEer/QyRO1/e3cXrevWiWt8tAVuBBCRIoCuBBCRIoCuBBCRIoCuBBCRIoCuBBCRMotVShmthnAHwEYBlAG8EwI4b+YWR+APwWwDcBZAJ8MIfCEDKtgco6rQI6mh6n98X5eWgoAFgs8/8H5+SF/As5H3PwTPJdD7pqvFECZ6yqsvUjtbe08vwPgqw4+tfFVt83YUi8fJ8nVJl8/86jbV0czb3Myzed1YabH7asc+HFJmJ/XpRY02rdnl7hC48Q8VxI90OkrJzIFnucnV/JP8YF2rlDpvsdXP3lM7ebn6UyOK0emqoSeVJr7fUeVknJPjnLlyO7Wy9S+Le3nLxlI8uPynfkHqP3o3Ijb13yBK2pqxe1cgRcB/FYI4T4A7wPwOTO7H8DTAF4MIewC8GLlfyFiQr4touaWATyEMB5CeK3y9xyAYwBGATwJ4LnKy54D8Ik1mqMQa4J8W8TOsu6Bm9k2AA8D+CGAoRDCOHD9RADgP0EixDpHvi1i5LYDuJl1APhLAF8IIWSW0e6AmR0ys0OlzOJK5ijEmlIL316a5fmwhVhLbiuAm1ka1x386yGEv6qYJ8xspLJ9BADNQh9CeCaEsD+EsD/ZxR+3FaJR1Mq3m7r9FApCrBW3DOBmZgC+CuBYCOHLN2z6FoCnKn8/BeCF2k9PiLVDvi1i53aSWX0QwL8F8IaZHa7Y/gOA3wPwZ2b2WQDnAfzKmszQ4dS1DdTe3eRLoD4wcJraz4/5Jd0wu7zSR1u3T7nb7u/lkqa+NJctnVzgkjwAKJb5Z+//nuYlpADg8MQotS8ucAnajmF/XzzOXutbdptk40qnrUvfnspxSd6ppO8Pe7suUPuZrN9meztPYtaW5DK+DSk/AdX4Uje1e1LQ3S08YRcADKa4XPCeNE++BQC70zxp1NsFfm5dKPKEXQDwZ7P7qf24IzeeyfvfvqolM6sFtwzgIYSX4SeG+5naTkeI+iHfFrGjJzGFECJSFMCFECJSFMCFECJSFMCFECJS1kVJtVry+pSfWGbLFr6K/dD2i26bNy7wsle5a1y5cek0VxAAwPkOvoqd7OWJoZqaecIsABjo5Kvru7p95cjcVae8k7OMd2bKV5SY0yadLlF7A5UmdwwXF3rcbX1N3B8GmvwEUG0JrjbpTvIH7gYcdQgAfKSdJ5PamnISlVW5dkwbTwiXNr882bNOKcFjWX7+jud8FYqXgGrBsZcbWPxPV+BCCBEpCuBCCBEpCuBCCBEpCuBCCBEpCuBCCBEpCuBCCBEpd5yMsFjya1L+zTle026ww0/Sc98mnoCqI82lf0cmfBlj3qkPGJyakIUTvtTp8hLfdn6rL/1LNHOJX7qJ1+RscuzVaJyg6s6nmlzt9WmeqKy72c9T3p3mid+2t/J6kVeKnW5fU862Hxj3ufNLfgK5sVyPu83jap6nql4q8xCXr1IrNCZ0BS6EEJGiAC6EEJGiAC6EEJGiAC6EEJGiAC6EEJFiIfCSR2symNkUgHOVfzcA4Mvd9eFuHv9O3fetIQS/htgaso58+059b+/28alv1zWA/38Dmx0KIfDicxr/jh17PYy/1ui91fj1Gk+3UIQQIlIUwIUQIlIaGcCfaeDYd/v4d/O+1wO9txq/LjTsHrgQQojVoVsoQggRKQ0J4Gb2cTM7bmYnzezpOo991szeMLPDZnaoDuN9zcwmzezIDbY+MztoZicqv3vrPP7vmNmlyjE4bGZPrNHYm83sH8zsmJm9aWa/WbHXbf/rSSP9ujK+fPsu8+26B3AzSwL4bwB+HsD9AD5tZvfXeRofDiHsq5Pc51kAH3+X7WkAL4YQdgF4sfJ/PccHgN+vHIN9IYTvrNHYRQC/FUK4D8D7AHyu8l7Xc//rwjrxa0C+DdxFvt2IK/BHAZwMIZwOISwB+CaAJxswj7oQQngJwLV3mZ8E8Fzl7+cAfKLO49eFEMJ4COG1yt9zAI4BGEUd97+O3FV+Dci314NvNyKAjwK4cMP/Fyu2ehEA/L2Z/djMDtRx3BsZCiGMA9cdAcBgA+bweTN7vfI1dM1vYZjZNgAPA/gh1sf+15pG+zUg336Hu8a3GxHAWVb6ekphPhhC+Clc/6r7OTP7l3Uce73wBwB2ANgHYBzAl9ZyMDPrAPCXAL4QQsis5VgNpNF+Dci3gbvMtxsRwC8C2HzD/5sAjNVr8BDCWOX3JIDncf2rb72ZMLMRAKj8nqzn4CGEiRBCKYRQBvCHWMNjYGZpXHfwr4cQ/qpibuj+rxEN9WtAvg3cfb7diAD+KoBdZrbdzJoAfArAt+oxsJm1m1nnO38D+FkAR6q3WhO+BeCpyt9PAXihnoO/42AVfhFrdAzMzAB8FcCxEMKXb9jU0P1fIxrm14B8+x3uOt8OIdT9B8ATAN4GcArAb9dx3HsA/KTy82Y9xgbwDVz/KlfA9au0zwLox/UV6hOV3311Hv9/AngDwOu47nAjazT247h+G+F1AIcrP0/Uc//r+dMov66MLd++C31bT2IKIUSk6ElMIYSIFAVwIYSIFAVwIYSIFAVwIYSIFAVwIYSIFAVwIYSIFAVwIYSIFAVwIYSIlP8LQC1Mpi09XMQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.subplot(121)\n",
    "plt.imshow(conv1.value[1][0])\n",
    "plt.subplot(122)\n",
    "plt.imshow(conv1.value[1][1])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAABoCAYAAADYZ7pcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAPuUlEQVR4nO3da4yc5XUH8P+Z2ZnZmb3N3n1ZYpa1iV1fuNkOhVSlhABtVUhLaamqVqpQUNukUltFaj5V6peq7Zc2anqJ1RK1UQmREhCOIMQ0CVitDL7EEDAYY7xrvOx6b7b36t25Pf2ATRf8nDOX5d19vP7/voDn7Pu+z5x558y77555HnHOgYiIwhVb6QEQEZGNhZqIKHAs1EREgWOhJiIKHAs1EVHgWKiJiAJXV8kPicj9AL4GIA7g35xzf2P9fFNrwrWvT3ljsyX/4x/G80k1ViwanysFPSYl85Bwcb1FMZ3OqbGe5AVzv/XiH9PAmTzGzxUFqD638aYGV9eZ9ca6MjPmeDriF9XYhaKe93P5BjVWdGIesy6mJz8VK6ixgoub+53LJ9TY/LvD4865TqC6/La0xd2aHv9+y523Y9NNaiw5qW8Xm8sbey3TOlunv30LGT1/hYy925Rxzk+fGK0ptwCQrMu4dDLrjRXr7dfbOAURSxXVWFf9tBpzsM9dKz6Zr1djuWn9vQQAqfP+8/5ifhK5wpz3oGULtYjEAfwTgM8DGARwSET2Oufe1LZpX5/CXz51kzd2cPoG83ivnP2UGpua1s+w0qieuPhF+wXJZ/UX+qafO63G/u76p8z93pjwn1277zsDoLbc1nVm0fPXf+yN/dGO/eZ4Hs0eU2PPzFynxp4c3q3Gphb0vANAV0Z/o/Q2TKixkYVmc79Hh9erseO/8Vengerzu6YngW/s9efhlbk+czz//D+fU2Mb9uoFt+FnQ/pOC/oHGQCUOlrV2MROPTZ+i/0B0LftfTX247v/vqbcAkA6mcXtm7/ojU1v1D/oAODsZ/QLsfRG/ZPwTza/qMYWSvqHPQDkjYuFH4xsVWOD+/X3EgD0ftd/3h84+e/qNpXc+tgN4KRz7pRzLgfgSQAPVrAdlcfcRov5jQ5zu4wqKdTrAZxZ9O/BS4/R0jG30WJ+o8PcLqNKCrXvvsEVvzuJyGMiclhEDs+ct+670SJV57Y0PbsMw1o1yuZ3cW4nJ/RbYHSFqs/dXGFuGYa1OlVSqAcBLL7p0gPgihtrzrk9zrmdzrmdja32vR/6UNW5jTUZf1Whjyub38W5bWm3/6BFH1H1uZusK/NXTFJVUqgPAdgkIr0ikgTwCIC90Q7rmsHcRov5jQ5zu4zKdn045woi8mUAP8QHbTiPO+f09gEA8y6Jty6u88Z+cGSHebzMaX1IHQN6q1fLU0fVmFtYMI8Z7+xUY2N36V0qDzz8h+Z+9+z8lvfx6Uu/INaS23Qyj+09/r/KH5vx5/yyW1+4V42tOaBvl5zUuw8W1tm/Pf1sW5caG9mq/6V/W9uwud+mtP2aAtXn1znBvNIJ8Phbd5jHWvsT/ZondU7/lX9qV48aW2ixr6PS4/qtmsyY/pq1vW6/7U+Vyt9qruXcdTFBMe3Pbyxvd6IUW/XbqQ0pvZ3wO+/vVGOpuN1V84sd76ixL33qJ2rsxQc2m/vdP7LL+3h+RH9dKuqjds49B+C5Sn6WqsPcRov5jQ5zu3z4zUQiosCxUBMRBY6FmogocCzURESBY6EmIgocCzURUeAqas+r1nQ+hZeGN3pjDf32IZ01k2m9Me3gQ7eosXLTnCanjH7UUb1Hc/5HjeZ+v77OP6PaaO579oAMuWIcpyf9M6MNPOvP+WW9b+u9x4kD6qRniK3tVmOlhB4DgPyA/nqPz+nb/vQW+zzZ0n5WjR0yt9RNl+rx4vQWb0xetWd3yzXqfcBnb9fPkzW/9p4au7uj3zzmE8f8/bgA0LovrcYWsvZskqlxO14rKZZQd84/BcLFLfa3Fu/Zrrdov9Svz2w4Otauxhr77W+ifjO7QY1dd+egGnthy/fN/fZ9drv38eLzeqHiFTURUeBYqImIAsdCTUQUOBZqIqLAsVATEQWOhZqIKHCRtOcVCnGMjvsXJ63L2tMZ5rv16Qxju/XpInvbzqmx9pS9Ksr+d/S2toajeptTw1m77+/VQf90kReNldbLKS7U4Xy/vz1v3Zg9ngt9+krahe23qrH6CWMl8Un7mO1v6i2Bk3PGyucNbeZ+R2765FcLmS0kceicvyWr9YS9+svYrfo1T6FJ3/YLRgveF1tfMY/ZskNfVf4b796nxuLz5m6Rbyqz+nmtikXI1Iw3NL7Lfr1/te01Nfajl/3tbgDQdVDfZ+tres0AgEJWX7j5vbg+Pe2+DfbUv7+51T8l87fS+uvJK2oiosCxUBMRBY6FmogocCzURESBY6EmIgocCzURUeAiac9DCXAL/pmpnD1hFWJJvZXp/g1vqbG7mvVYAnZr1WxBbxM7fG6TGovl7c+5wry/TceVap+dTApA/ag/ifOtdlvVxO166yOMMc0P6+1GTQP2c3Fx/QUvpvRtM8P2fsc2NpjxWizk63BquMMb29Rvt3gO3a3P/hab18+TyYLe/jletNu8/rztlBr7erN+LmTO2rm9uFGfMXJJnIPL+fd9z63mAua4o35EjTWe1vPbdFpv45QF+3nWTemx3qf099KLv+KfgfGyG+v9Mz/Wi9GabO6RiIhWHAs1EVHgWKiJiALHQk1EFDgWaiKiwLFQExEFLpr2PCeQBf9nQHrEbg3Kz+jtSqltBTW2KTGhxs4V9VmwAKAr5Z/RCwASa/T2npm8vSBnIu1vt5FY7bOTxXNA4xn/9uUWLc206rNzbe3WF4s91r5GjU1k7UVf4/P6mIppvW2yfszu45wZajHjNXGCktJyWWiyZzyUgv48rRwcu7BWjR3J6IurAsDNKf01cyl9VsPsu3a76syddjwKu5rthXxTol9TTt+o14XuI3ruS9kyLZ4lPYdS1N/DxTLXvwene72Pz5b0BaYrKtQiMgBgGkARQME5t7OS7ag85jZazG90mNvlU80V9S8558YjG8m1jbmNFvMbHeZ2GfAeNRFR4Cot1A7APhE5IiKP+X5ARB4TkcMicrg4o9/zpStUldvCvP1VZrqCmd+PnLfTzG2Vqjp3c6UyS8uQqtJbH3c654ZEpAvACyJy3Dm3f/EPOOf2ANgDAKkN10W0ls+qVFVuGzqY2yqZ+f3Iedvbw9xWp6pztyXRyfzWqKIraufc0KX/jgJ4GsDuKAd1LWFuo8X8Roe5XT5lC7WINIhI0+X/B3AvgDeiHti1gLmNFvMbHeZ2eVVy66MbwNMicvnnn3DOPW9uEXNwGX8vZnLS/mxIj+qxvQPb1NgDLT9VYzuMqVMBYEfjGTX2Rqve5zpUZrrSQk6Z6vX/fwGsOrfigLoF/2+QRb1NGgCQd/p4d2UH1NjNzYNqbNsOPXflfOXww2osdcLucXXxiu7aVZXfuroiOrv8c1te2Oif/vTDbY3b200Deqw/4V+pHgCeNI8IPDum3/PtPKD3oWdODJv7zWT07zIsUvW56wpFFCf8K3//6zu/YB7s0dv0c/Bv7/6OGvuL2G+psboL9vcrGgaNaXhH9B7rrWl9rADwD2c+5318Nq/36pc9251zpwDcVO7nqHrMbbSY3+gwt8uL7XlERIFjoSYiChwLNRFR4FioiYgCx0JNRBS4SKY5TSQKWLfO34Zzfo0+ZSYAdB/SVwYuPZNVY7999ktqrHnNtHnM2Vm9Tac0nlJjsZzdnieifBGrzOrl5j6LDokZf2tQrsGeGrT4tj4l6fEevQ3xaz3/rcYaY3aL011vfEGNXf+P+nZz6+yWyrnuT/4ao7t+Cl/ZtM8b++pnHzK3rXtfP09E7+TChuf0ladzL+mvCQDMjuj9mK0HD6ixqV//jLnfDa2n1VhUjdJuX7sZfzh7nxr7zxu+r8bid+mtey/P9JnH/O7R29TY7IS+Qnxf0ugxBjB31P9cS3N6OeYVNRFR4FioiYgCx0JNRBQ4FmoiosCxUBMRBY6FmogocJG05zUn5nHP2re9sf/aYq8ePTOkt3t1HtCXZuv83oi+05TeOgUAaNPHNLNZb8O50Gen7+JOpX0qYfRrlVGqE8y3+dvwMuN2S1uD0TV08PwONXZb63Y11nzSPCTa/uOgEdVn3pv6M7uFLN9cew41LbE87sv4V/Y+tP2oue0zST1/FxL6avUNw3pLZXrQXnEmNqO35038/s+rsZkH/TMEXnZP9j019qy5ZRkiEOW9uPbbx81NB6c/rcZuf/BRNfbL1+sre/fP2i2BMJY5uOE2/dz90zcfMXe75hV/S+bwrH5AXlETEQWOhZqIKHAs1EREgWOhJiIKHAs1EVHgWKiJiALHQk1EFLhI+qjrY3lsqR/yxn53m9VXC3w7rk8tmG/sVGMdr+tTeMbm7f7iqT591eVzW/WpTFObL5j7/YO+I97H/yVt98daXBzINfvHlJyxp11tPD6hxjI/1lemdjl9Kk6X16elBYD4jfpUkqcf6lZjF8tMcyp5+7nWYqYUx4H5rDf2e236tKEAsLBJfyulPl1QYwe296qxgSl7JfaGtH7MmztfV2N3tNjN73FrXtYlkHgMsaZGb8zN6T3hANDx8pgaG4vpdeHZHr2fPJe1n6ck9L7mk6/1qLG1/2s0YAPI9PungI4t6Oc8r6iJiALHQk1EFDgWaiKiwLFQExEFjoWaiChwLNRERIET5+xWkpp2KjIG4PJSxh0A9PlJV8ZKj2mDc07vKTJ8LLfAyj+XjwthPDXll7mtyCd17obwXD5upcek5jaSQv2RA4gcds7tjPQgVQpxTLUK7bmENp6lCO25hDaepQjxuYQ4pst464OIKHAs1EREgVuOQr1nGY5RrRDHVKvQnkto41mK0J5LaONZihCfS4hjArAM96iJiGhpeOuDiChwkRZqEblfRN4WkZMi8tUoj1XheAZE5HUReVVEDq/0eJYitNwCqye/zG20Qsvv1ZDbyG59iEgcwAkAnwcwCOAQgN9xzunrt0dMRAYA7HTOhda/WZUQc3tpXAO4yvPL3EYrxPxeDbmN8op6N4CTzrlTzrkcgCcBPBjh8a4lzG10mNtoMb81iLJQrwdwZtG/By89tpIcgH0ickREHlvhsSxFiLkFVkd+mdtohZjf4HMbyQovl/iW4FjpFpM7nXNDItIF4AUROe6c27/CY6pFiLkFVkd+mdtohZjf4HMb5RX1IIDrFv27B4B/fa5l4pwbuvTfUQBP44Nfw65GweUWWDX5ZW6jFVx+r4bcRlmoDwHYJCK9IpIE8AiAvREezyQiDSLSdPn/AdwL4I2VGs8SBZVbYFXll7mNVlD5vVpyG9mtD+dcQUS+DOCHAOIAHnfOHYvqeBXoBvC0iAAfPO8nnHPPr+B4ahZgboFVkl/mNloB5veqyC2/mUhEFDh+M5GIKHAs1EREgWOhJiIKHAs1EVHgWKiJiALHQk1EFDgWaiKiwLFQExEF7v8Ab+YOrQCD61wAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 4 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.subplot(141)\n",
    "plt.imshow(conv2.value[1][0])\n",
    "plt.subplot(142)\n",
    "plt.imshow(conv2.value[1][1])\n",
    "plt.subplot(143)\n",
    "plt.imshow(conv2.value[1][2])\n",
    "plt.subplot(144)\n",
    "plt.imshow(conv2.value[1][3])\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}