Tutorial/PyTorch Computer Vision.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Table of Contents\n",
    "\n",
    "# 1. Load data\n",
    "\n",
    "# 2. Prepare data\n",
    "\n",
    "# 3. Building a baseline model\n",
    "\n",
    "# 4. Making predictions and evaluating\n",
    "\n",
    "# 5. Setup device\n",
    "\n",
    "# 6. Adding non-linearity\n",
    "\n",
    "# 7. Convolutional Neural Network\n",
    "\n",
    "# 8. Comparing our models\n",
    "\n",
    "# 9. Evaluating our best model\n",
    "\n",
    "# 10. Confusion matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### When it comes to computer vision problems, PyTorch has several libraries that you need to know such as torchvision and its sub libraries.\n",
    "##### Let us import the necessary libraries first."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch \n",
    "from torch import nn \n",
    "\n",
    "import torchvision\n",
    "from torchvision import datasets\n",
    "from torchvision.transforms import ToTensor\n",
    "\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Load data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### For this notebook, we will be using the FashionMNIST dataset.\n",
    "##### It contains grayscale images of 10 differnt kinds of clothing.\n",
    "##### So it is a multi-class classification task. \n",
    "##### Let us download the dataset using torchvision.datasets.FashionMNIST()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Training data\n",
    "train_data = datasets.FashionMNIST(\n",
    "    root='data',\n",
    "    train=True,\n",
    "    download=True,\n",
    "    transform=ToTensor(),\n",
    "    target_transform=None\n",
    ")\n",
    "\n",
    "# Testing data\n",
    "test_data = datasets.FashionMNIST(\n",
    "    root='data',\n",
    "    train=False,\n",
    "    download=True,\n",
    "    transform=ToTensor()\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Let's check out the first sample of the training data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[[0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0039, 0.0000, 0.0000, 0.0510,\n",
       "           0.2863, 0.0000, 0.0000, 0.0039, 0.0157, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0039, 0.0039, 0.0000],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0118, 0.0000, 0.1412, 0.5333,\n",
       "           0.4980, 0.2431, 0.2118, 0.0000, 0.0000, 0.0000, 0.0039, 0.0118,\n",
       "           0.0157, 0.0000, 0.0000, 0.0118],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0235, 0.0000, 0.4000, 0.8000,\n",
       "           0.6902, 0.5255, 0.5647, 0.4824, 0.0902, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0471, 0.0392, 0.0000],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.6078, 0.9255,\n",
       "           0.8118, 0.6980, 0.4196, 0.6118, 0.6314, 0.4275, 0.2510, 0.0902,\n",
       "           0.3020, 0.5098, 0.2824, 0.0588],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0039, 0.0000, 0.2706, 0.8118, 0.8745,\n",
       "           0.8549, 0.8471, 0.8471, 0.6392, 0.4980, 0.4745, 0.4784, 0.5725,\n",
       "           0.5529, 0.3451, 0.6745, 0.2588],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0039, 0.0039, 0.0039, 0.0000, 0.7843, 0.9098, 0.9098,\n",
       "           0.9137, 0.8980, 0.8745, 0.8745, 0.8431, 0.8353, 0.6431, 0.4980,\n",
       "           0.4824, 0.7686, 0.8980, 0.0000],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.7176, 0.8824, 0.8471,\n",
       "           0.8745, 0.8941, 0.9216, 0.8902, 0.8784, 0.8706, 0.8784, 0.8667,\n",
       "           0.8745, 0.9608, 0.6784, 0.0000],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.7569, 0.8941, 0.8549,\n",
       "           0.8353, 0.7765, 0.7059, 0.8314, 0.8235, 0.8275, 0.8353, 0.8745,\n",
       "           0.8627, 0.9529, 0.7922, 0.0000],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0039, 0.0118, 0.0000, 0.0471, 0.8588, 0.8627, 0.8314,\n",
       "           0.8549, 0.7529, 0.6627, 0.8902, 0.8157, 0.8549, 0.8784, 0.8314,\n",
       "           0.8863, 0.7725, 0.8196, 0.2039],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0235, 0.0000, 0.3882, 0.9569, 0.8706, 0.8627,\n",
       "           0.8549, 0.7961, 0.7765, 0.8667, 0.8431, 0.8353, 0.8706, 0.8627,\n",
       "           0.9608, 0.4667, 0.6549, 0.2196],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0157, 0.0000, 0.0000, 0.2157, 0.9255, 0.8941, 0.9020,\n",
       "           0.8941, 0.9412, 0.9098, 0.8353, 0.8549, 0.8745, 0.9176, 0.8510,\n",
       "           0.8510, 0.8196, 0.3608, 0.0000],\n",
       "          [0.0000, 0.0000, 0.0039, 0.0157, 0.0235, 0.0275, 0.0078, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.9294, 0.8863, 0.8510, 0.8745,\n",
       "           0.8706, 0.8588, 0.8706, 0.8667, 0.8471, 0.8745, 0.8980, 0.8431,\n",
       "           0.8549, 1.0000, 0.3020, 0.0000],\n",
       "          [0.0000, 0.0118, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.2431, 0.5686, 0.8000, 0.8941, 0.8118, 0.8353, 0.8667,\n",
       "           0.8549, 0.8157, 0.8275, 0.8549, 0.8784, 0.8745, 0.8588, 0.8431,\n",
       "           0.8784, 0.9569, 0.6235, 0.0000],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000, 0.0706, 0.1725, 0.3216, 0.4196,\n",
       "           0.7412, 0.8941, 0.8627, 0.8706, 0.8510, 0.8863, 0.7843, 0.8039,\n",
       "           0.8275, 0.9020, 0.8784, 0.9176, 0.6902, 0.7373, 0.9804, 0.9725,\n",
       "           0.9137, 0.9333, 0.8431, 0.0000],\n",
       "          [0.0000, 0.2235, 0.7333, 0.8157, 0.8784, 0.8667, 0.8784, 0.8157,\n",
       "           0.8000, 0.8392, 0.8157, 0.8196, 0.7843, 0.6235, 0.9608, 0.7569,\n",
       "           0.8078, 0.8745, 1.0000, 1.0000, 0.8667, 0.9176, 0.8667, 0.8275,\n",
       "           0.8627, 0.9098, 0.9647, 0.0000],\n",
       "          [0.0118, 0.7922, 0.8941, 0.8784, 0.8667, 0.8275, 0.8275, 0.8392,\n",
       "           0.8039, 0.8039, 0.8039, 0.8627, 0.9412, 0.3137, 0.5882, 1.0000,\n",
       "           0.8980, 0.8667, 0.7373, 0.6039, 0.7490, 0.8235, 0.8000, 0.8196,\n",
       "           0.8706, 0.8941, 0.8824, 0.0000],\n",
       "          [0.3843, 0.9137, 0.7765, 0.8235, 0.8706, 0.8980, 0.8980, 0.9176,\n",
       "           0.9765, 0.8627, 0.7608, 0.8431, 0.8510, 0.9451, 0.2549, 0.2863,\n",
       "           0.4157, 0.4588, 0.6588, 0.8588, 0.8667, 0.8431, 0.8510, 0.8745,\n",
       "           0.8745, 0.8784, 0.8980, 0.1137],\n",
       "          [0.2941, 0.8000, 0.8314, 0.8000, 0.7569, 0.8039, 0.8275, 0.8824,\n",
       "           0.8471, 0.7255, 0.7725, 0.8078, 0.7765, 0.8353, 0.9412, 0.7647,\n",
       "           0.8902, 0.9608, 0.9373, 0.8745, 0.8549, 0.8314, 0.8196, 0.8706,\n",
       "           0.8627, 0.8667, 0.9020, 0.2627],\n",
       "          [0.1882, 0.7961, 0.7176, 0.7608, 0.8353, 0.7725, 0.7255, 0.7451,\n",
       "           0.7608, 0.7529, 0.7922, 0.8392, 0.8588, 0.8667, 0.8627, 0.9255,\n",
       "           0.8824, 0.8471, 0.7804, 0.8078, 0.7294, 0.7098, 0.6941, 0.6745,\n",
       "           0.7098, 0.8039, 0.8078, 0.4510],\n",
       "          [0.0000, 0.4784, 0.8588, 0.7569, 0.7020, 0.6706, 0.7176, 0.7686,\n",
       "           0.8000, 0.8235, 0.8353, 0.8118, 0.8275, 0.8235, 0.7843, 0.7686,\n",
       "           0.7608, 0.7490, 0.7647, 0.7490, 0.7765, 0.7529, 0.6902, 0.6118,\n",
       "           0.6549, 0.6941, 0.8235, 0.3608],\n",
       "          [0.0000, 0.0000, 0.2902, 0.7412, 0.8314, 0.7490, 0.6863, 0.6745,\n",
       "           0.6863, 0.7098, 0.7255, 0.7373, 0.7412, 0.7373, 0.7569, 0.7765,\n",
       "           0.8000, 0.8196, 0.8235, 0.8235, 0.8275, 0.7373, 0.7373, 0.7608,\n",
       "           0.7529, 0.8471, 0.6667, 0.0000],\n",
       "          [0.0078, 0.0000, 0.0000, 0.0000, 0.2588, 0.7843, 0.8706, 0.9294,\n",
       "           0.9373, 0.9490, 0.9647, 0.9529, 0.9569, 0.8667, 0.8627, 0.7569,\n",
       "           0.7490, 0.7020, 0.7137, 0.7137, 0.7098, 0.6902, 0.6510, 0.6588,\n",
       "           0.3882, 0.2275, 0.0000, 0.0000],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.1569,\n",
       "           0.2392, 0.1725, 0.2824, 0.1608, 0.1373, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "           0.0000, 0.0000, 0.0000, 0.0000]]]),\n",
       " 9)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "image, label = train_data[0]\n",
    "image, label"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Let's take a look at the shape of this huge tensor."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 28, 28])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "image.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### This means we have one color channel with 28 pixels on both of its axes.\n",
    "##### The order of the shape can be interpreted as **Color Channels, Height, Width**."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000, 60000, 10000, 10000)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_data.data), len(train_data.targets), len(test_data.data), len(test_data.targets)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### We have 60,000 training data in total and 10,000 testing data in total."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['T-shirt/top',\n",
       " 'Trouser',\n",
       " 'Pullover',\n",
       " 'Dress',\n",
       " 'Coat',\n",
       " 'Sandal',\n",
       " 'Shirt',\n",
       " 'Sneaker',\n",
       " 'Bag',\n",
       " 'Ankle boot']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class_names = train_data.classes\n",
    "class_names"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### And these are the classes we have, 10 in total."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### We can also visualize what some of these pictures might look like."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Image shape: torch.Size([1, 28, 28])\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaAAAAGzCAYAAABpdMNsAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy88F64QAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAkhklEQVR4nO3de3SU9b3v8c/kNgSYTAghNwkYUEAFYkshplhESYG0xwPK7tHWswo9Li0YXEXarQu3ilq70+La1lOLes7aLdS1xNuqyJZtOVVogrQJyu1QaptCGgUlCRfNTMh1kvmdPzhGI9ffwyS/JLxfa81aZOb58Px4eJJPnszMNz5jjBEAAL0szvUCAAAXJwoIAOAEBQQAcIICAgA4QQEBAJyggAAATlBAAAAnKCAAgBMUEADACQoIAOAEBQT0kp07d2ru3LlKSUlRIBDQ7NmztWfPHtfLApzxMQsO6Hm7du3S9OnTlZubq+9///uKRqN6+umn9fHHH+udd97R+PHjXS8R6HUUENALvvnNb6qiokL79+/X8OHDJUm1tbUaN26cZs+erd/+9reOVwj0Pn4EB/SCt99+W0VFRV3lI0nZ2dm67rrrtHHjRp04ccLh6gA3KCCgF7S1tSk5OfmU+wcPHqz29nbt27fPwaoAtyggoBeMHz9elZWV6uzs7Lqvvb1d27dvlyR99NFHrpYGOEMBAb3grrvu0t///nfdfvvteu+997Rv3z5997vfVW1trSSppaXF8QqB3kcBAb1g8eLFuv/++7Vu3TpdddVVmjRpkqqrq3XvvfdKkoYOHep4hUDvo4CAXvKTn/xE9fX1evvtt7V37169++67ikajkqRx48Y5Xh3Q+3gZNuDQtGnTVFtbqw8++EBxcXw/iIsLZzzgyEsvvaR3331Xy5Yto3xwUeIKCOgFW7du1aOPPqrZs2dr+PDhqqys1Jo1a/T1r39dr7/+uhISElwvEeh1nPVAL7jkkksUHx+vxx9/XI2NjcrLy9Njjz2m5cuXUz64aHEFBABwgh88AwCcoIAAAE5QQAAAJyggAIATFBAAwAkKCADgRJ97A0I0GtXhw4cVCATk8/lcLwcAYMkYo8bGRuXk5Jx1ykefK6DDhw8rNzfX9TIAABfo0KFDGjly5Bkf73MFFAgEJEnX6htKUKLj1QAAbHUoom16o+vr+Zn0WAGtXr1ajz/+uOrq6pSfn6+nnnpK06ZNO2fu0x+7JShRCT4KCAD6nf8/X+dcT6P0yIsQXnrpJS1fvlwrV67Url27lJ+frzlz5ujIkSM9sTsAQD/UIwX0xBNP6I477tD3vvc9XXnllXr22Wc1ePBg/frXv+6J3QEA+qGYF1B7e7t27typoqKiz3YSF6eioiJVVFScsn1bW5vC4XC3GwBg4It5AR07dkydnZ3KzMzsdn9mZqbq6upO2b60tFTBYLDrxivgAODi4PyNqCtWrFAoFOq6HTp0yPWSAAC9IOavgktPT1d8fLzq6+u73V9fX6+srKxTtvf7/fL7/bFeBgCgj4v5FVBSUpKmTJmizZs3d90XjUa1efNmFRYWxnp3AIB+qkfeB7R8+XItXLhQX/nKVzRt2jQ9+eSTampq0ve+972e2B0AoB/qkQK65ZZbdPToUT300EOqq6vT1VdfrU2bNp3ywgQAwMXLZ4wxrhfxeeFwWMFgUDM1j0kIANAPdZiIyrRBoVBIKSkpZ9zO+avgAAAXJwoIAOAEBQQAcIICAgA4QQEBAJyggAAATlBAAAAnKCAAgBMUEADACQoIAOAEBQQAcIICAgA4QQEBAJyggAAATlBAAAAnKCAAgBMUEADACQoIAOAEBQQAcIICAgA4QQEBAJyggAAATlBAAAAnKCAAgBMUEADACQoIAOBEgusFAH2Kz2efMSb26ziN+OFp1plP5ozztK+UdZWectY8HG9fQqJ1xkTarTN9npdz1aseOse5AgIAOEEBAQCcoIAAAE5QQAAAJyggAIATFBAAwAkKCADgBAUEAHCCAgIAOEEBAQCcoIAAAE5QQAAAJxhGCnyOLz7eOmM6OqwzcVdfaZ356/eH2u+nxToiSUpsmmadSWiJ2u/n9zusM706WNTLsFQP55B89tcCvXkcfAl2VeEzRjqPTwuugAAATlBAAAAnKCAAgBMUEADACQoIAOAEBQQAcIICAgA4QQEBAJyggAAATlBAAAAnKCAAgBMUEADACYaRAp9jO3RR8jaM9NCcVOvMbYVvW2f+eHSMdUaSPvBnWWdMsv1+EooKrTPjnv7IOtPx/kHrjCTJGPuIh/PBi/hhw7wFOzvtI+Gw1fbGnN8x4AoIAOAEBQQAcCLmBfTwww/L5/N1u02YMCHWuwEA9HM98hzQVVddpbfeeuuznXj4uToAYGDrkWZISEhQVpb9k5gAgItHjzwHtH//fuXk5GjMmDG67bbbdPDgmV+B0tbWpnA43O0GABj4Yl5ABQUFWrt2rTZt2qRnnnlGNTU1+trXvqbGxsbTbl9aWqpgMNh1y83NjfWSAAB9UMwLqLi4WN/61rc0efJkzZkzR2+88YYaGhr08ssvn3b7FStWKBQKdd0OHToU6yUBAPqgHn91QGpqqsaNG6cDBw6c9nG/3y+/39/TywAA9DE9/j6gEydOqLq6WtnZ2T29KwBAPxLzAvrRj36k8vJyvf/++/rTn/6km266SfHx8fr2t78d610BAPqxmP8I7sMPP9S3v/1tHT9+XCNGjNC1116ryspKjRgxIta7AgD0YzEvoBdffDHWfyXQa6Ktrb2yn/YvnbDO/FNwh3VmUFzEOiNJ5XFR68xHW+xfwdo52f44fPBEwDoT3f1V64wkDd9nP7gzZXetdebYjEusM0en2A9KlaTMSvvMsLeqrbY30Xbp2Lm3YxYcAMAJCggA4AQFBABwggICADhBAQEAnKCAAABOUEAAACcoIACAExQQAMAJCggA4AQFBABwggICADjR47+QDnDC5/OWM/YDHk/8t2usM9+9ssw6Ux2xnyg/Mulj64wkfStnp33ov9tnfll1nXWm6R9B60zcEG+DO+uusf8e/aN59v9PJtJhnRm2y9uX77iF9daZcPsYq+07Iq3ShvNYi/VKAACIAQoIAOAEBQQAcIICAgA4QQEBAJyggAAATlBAAAAnKCAAgBMUEADACQoIAOAEBQQAcIICAgA4QQEBAJxgGjZ6l9cp1X3YNfe9Y525fuh7PbCSU10ib1Ogm0ySdaahc4h1ZuWV/2mdOTouYJ2JGG9f6v59/1etMyc8TOuO77D/vLjmf+y2zkjSgrR3rTOrfjvJavsOEzmv7bgCAgA4QQEBAJyggAAATlBAAAAnKCAAgBMUEADACQoIAOAEBQQAcIICAgA4QQEBAJyggAAATlBAAAAnGEaK3mW8Dcfsy/afyLDOHE8Zap2p60i1zgyPP2GdkaRAXIt15tLEY9aZo532g0XjE6PWmXYTb52RpEeuet0603pFonUm0ddpnfnqoMPWGUn61nvftc4M0T887etcuAICADhBAQEAnKCAAABOUEAAACcoIACAExQQAMAJCggA4AQFBABwggICADhBAQEAnKCAAABOUEAAACcYRgpcoBF++4Gfg3wR60ySr8M6czgyzDojSftbxltn/h62H8o6N/Mv1pmIh8Gi8fI2BNfLkNCcxE+sM63GfoCp/Rl00vRM+8Giezzu61y4AgIAOEEBAQCcsC6grVu36sYbb1ROTo58Pp9ee+21bo8bY/TQQw8pOztbycnJKioq0v79+2O1XgDAAGFdQE1NTcrPz9fq1atP+/iqVav0i1/8Qs8++6y2b9+uIUOGaM6cOWptbb3gxQIABg7rFyEUFxeruLj4tI8ZY/Tkk0/qgQce0Lx58yRJzz33nDIzM/Xaa6/p1ltvvbDVAgAGjJg+B1RTU6O6ujoVFRV13RcMBlVQUKCKiorTZtra2hQOh7vdAAADX0wLqK6uTpKUmZnZ7f7MzMyux76otLRUwWCw65abmxvLJQEA+ijnr4JbsWKFQqFQ1+3QoUOulwQA6AUxLaCsrCxJUn19fbf76+vrux77Ir/fr5SUlG43AMDAF9MCysvLU1ZWljZv3tx1Xzgc1vbt21VYWBjLXQEA+jnrV8GdOHFCBw4c6Pq4pqZGe/bsUVpamkaNGqVly5bpscce0+WXX668vDw9+OCDysnJ0fz582O5bgBAP2ddQDt27ND111/f9fHy5cslSQsXLtTatWt17733qqmpSXfeeacaGhp07bXXatOmTRo0aFDsVg0A6Pd8xhhvU/p6SDgcVjAY1EzNU4LPfkAf+jifzz4Sbz980nTYD+6UpPhh9sM7b634s/1+fPafdkc7AtaZ1Phm64wklTfYDyP9y/HTP897No+O/w/rzK7mS60zOUn2A0Ilb8fv/fZ068zl/tO/SvhsfvdJvnVGknIHfWyd+f2yGVbbd3S0alvZIwqFQmd9Xt/5q+AAABcnCggA4AQFBABwggICADhBAQEAnKCAAABOUEAAACcoIACAExQQAMAJCggA4AQFBABwggICADhBAQEAnLD+dQzABfEwfN2XYH+aep2Gfej2K6wzNwx+3Trzp9ZLrDMjEhqtMxFjP0lckrL9IetMILPVOtPQOdg6k5ZwwjrT2JlsnZGkwXFt1hkv/09fTjpmnbnnrS9bZyQpMPG4dSYl0e5aJXqe1zZcAQEAnKCAAABOUEAAACcoIACAExQQAMAJCggA4AQFBABwggICADhBAQEAnKCAAABOUEAAACcoIACAEwwjRa/yJSZZZ6Kt9kMuvUr/c7t15lhnonUmNa7ZOpPk67TOtHscRvrVtBrrzFEPAz93teRZZwLxLdaZEXH2A0IlKTfRfnDnn1tzrTNvNF1mnbn9v7xlnZGkF/73160zSZv+ZLV9nImc33bWKwEAIAYoIACAExQQAMAJCggA4AQFBABwggICADhBAQEAnKCAAABOUEAAACcoIACAExQQAMAJCggA4MTFPYzU5/MWS7AfPumL99D1cfaZaGub/X6i9kMuvTIR+2Gfvel//q9fWmcOdaRaZ+oi9pnUePsBpp3ydo5XtgStM4Pizm8A5eeNSAhbZ8JR+6GnXjVGB1lnIh4GwHo5dvcN32+dkaRXQ0Wecj2BKyAAgBMUEADACQoIAOAEBQQAcIICAgA4QQEBAJyggAAATlBAAAAnKCAAgBMUEADACQoIAOAEBQQAcGLADCP1Jdj/U0xHh6d9eRmoaexnDQ5ILfOmWWcOzbcflnrbl96xzkhSXUfAOrO7+VLrTDC+xTozJM5+0GyrsR+cK0mH24dZZ7wM1ExLOGGdyfAwwLTTePte+6OI/XHwwsug2Q877I+dJDX+10brTOpznnZ1TlwBAQCcoIAAAE5YF9DWrVt14403KicnRz6fT6+99lq3xxctWiSfz9ftNnfu3FitFwAwQFgXUFNTk/Lz87V69eozbjN37lzV1tZ23V544YULWiQAYOCxfua+uLhYxcXFZ93G7/crKyvL86IAAANfjzwHVFZWpoyMDI0fP15LlizR8ePHz7htW1ubwuFwtxsAYOCLeQHNnTtXzz33nDZv3qyf/exnKi8vV3FxsTo7T/9S2tLSUgWDwa5bbm5urJcEAOiDYv4+oFtvvbXrz5MmTdLkyZM1duxYlZWVadasWadsv2LFCi1fvrzr43A4TAkBwEWgx1+GPWbMGKWnp+vAgQOnfdzv9yslJaXbDQAw8PV4AX344Yc6fvy4srOze3pXAIB+xPpHcCdOnOh2NVNTU6M9e/YoLS1NaWlpeuSRR7RgwQJlZWWpurpa9957ry677DLNmTMnpgsHAPRv1gW0Y8cOXX/99V0ff/r8zcKFC/XMM89o7969+s1vfqOGhgbl5ORo9uzZ+vGPfyy/3x+7VQMA+j2fMca4XsTnhcNhBYNBzdQ8Jfi8DVLsixKy7d8XFcnLtM58fMVg60xzls86I0lXf+Ov1plFmdusM0c77Z8XTPR5GzTb2JlsnclKbLDObAldaZ0ZmmA/jNTL0FNJ+nLy+9aZhqj9uZeT8Il15r4D/2SdyRxsP4BTkv599BvWmYiJWmeqIvbfoAfi7IciS9LbzZdZZ9ZfOcJq+w4TUZk2KBQKnfV5fWbBAQCcoIAAAE5QQAAAJyggAIATFBAAwAkKCADgBAUEAHCCAgIAOEEBAQCcoIAAAE5QQAAAJyggAIATFBAAwImY/0puV9qKp1pnMv7lH572dXXKh9aZK5Ptp0C3Ru2ngQ+Ki1hn3mu5xDojSc3RJOvM/nb7qeChDvspy/E++4nEknSkPWCd+beaIuvM5mnPWmceODzXOhOX7G3Y/fHOodaZBUPDHvZkf45/f9RW68yYpCPWGUna2GT/izQPR4ZZZzITQ9aZSxOPWmck6ebA360z62U3Dft8cQUEAHCCAgIAOEEBAQCcoIAAAE5QQAAAJyggAIATFBAAwAkKCADgBAUEAHCCAgIAOEEBAQCcoIAAAE702WGkvoQE+Xznv7yCf33Xeh+zAn+xzkhSs/FbZ7wMFvUy1NCLYEKzp1xbxP70ORJJ8bQvW+P8dZ5yN6Xssc5s/WWBdeba1rutM9U3rLHObG6Jt85I0tEO+/+nW2tusM7sOphrnbnm0hrrzKTAR9YZydsg3EB8q3Um0ddhnWmK2n8dkqTKVvtBsz2FKyAAgBMUEADACQoIAOAEBQQAcIICAgA4QQEBAJyggAAATlBAAAAnKCAAgBMUEADACQoIAOAEBQQAcKLPDiOtXTJF8f5B5739w8GnrPex7uNrrDOSlDvoY+vM6KRj1pn85A+sM14E4uyHJ0rS+BT7AYobm0ZaZ8oaJlhnshMbrDOS9HbzWOvMiw8/bp1ZdM8PrTOFbyy2zoQv9fY9ZscQY51JyT9unXngS/9pnUnydVpnGjrth4pKUpq/yTqTGu9tuK8tL0ORJSkQ12KdiR9/mdX2prNN2n/u7bgCAgA4QQEBAJyggAAATlBAAAAnKCAAgBMUEADACQoIAOAEBQQAcIICAgA4QQEBAJyggAAATlBAAAAn+uww0sFHoopPip739hvDV1vvY0zyUeuMJB2LBKwz/+fEJOvMyORPrDPBePtBg5f566wzkrSnNdU6s+noVdaZnOSwdaY+ErTOSNLxyBDrTHPUfijkr37+hHXm3+qLrDM3pe2yzkhSfpL9YNGGqP33s++1Z1lnGqPnP6T4U60m0TojSSEPQ0wDHj4HI8b+S3G8Of+vj5+XGmc/LDU8abjV9h2RVoaRAgD6LgoIAOCEVQGVlpZq6tSpCgQCysjI0Pz581VVVdVtm9bWVpWUlGj48OEaOnSoFixYoPr6+pguGgDQ/1kVUHl5uUpKSlRZWak333xTkUhEs2fPVlPTZ7+06Z577tHrr7+uV155ReXl5Tp8+LBuvvnmmC8cANC/WT3ztWnTpm4fr127VhkZGdq5c6dmzJihUCikX/3qV1q3bp1uuOEGSdKaNWt0xRVXqLKyUtdc4+03kAIABp4Leg4oFApJktLS0iRJO3fuVCQSUVHRZ6/WmTBhgkaNGqWKiorT/h1tbW0Kh8PdbgCAgc9zAUWjUS1btkzTp0/XxIkTJUl1dXVKSkpSampqt20zMzNVV3f6l/qWlpYqGAx23XJzc70uCQDQj3guoJKSEu3bt08vvvjiBS1gxYoVCoVCXbdDhw5d0N8HAOgfPL0RdenSpdq4caO2bt2qkSNHdt2flZWl9vZ2NTQ0dLsKqq+vV1bW6d9w5vf75ffbv5EPANC/WV0BGWO0dOlSrV+/Xlu2bFFeXl63x6dMmaLExERt3ry5676qqiodPHhQhYWFsVkxAGBAsLoCKikp0bp167RhwwYFAoGu53WCwaCSk5MVDAZ1++23a/ny5UpLS1NKSoruvvtuFRYW8go4AEA3VgX0zDPPSJJmzpzZ7f41a9Zo0aJFkqSf//zniouL04IFC9TW1qY5c+bo6aefjsliAQADh88YY1wv4vPC4bCCwaBmXPugEhLOf+jg1Cd3Wu9rXzjHOiNJmYMarTOTh35onalqth/UeLglxTozOCFinZGk5Hj7XIexf91Lht/+eI/y2w/TlKRAnP0gySRfp3Wm08Prf65KOmydOdgxzDojSXUdqdaZ95rtP5+GJdgPxvyzh8/b5o4k64wktXXaP03e2mGfCfpbrTNT0z6wzkhSnOy/5K/7j+usto+2tuofj/2LQqGQUlLO/DWJWXAAACcoIACAExQQAMAJCggA4AQFBABwggICADhBAQEAnKCAAABOUEAAACcoIACAExQQAMAJCggA4AQFBABwwtNvRO0Ncdv2Ks6XeN7bv/L76db7eHDeK9YZSSpvmGCd2Vg3yToTbrf/TbEjBjdZZ1IS7adNS1Jaov2+gh6mHw/ydVhnPukYYp2RpLa48z/nPtUpn3Wmri1onflj9HLrTCQab52RpDYPOS/T0T9uT7fO5CSHrDONHec/Wf/z3m9Ms84cCw21zrQOtv9SvK1zrHVGkuZm/cU6k3zE7hzvbDu/7bkCAgA4QQEBAJyggAAATlBAAAAnKCAAgBMUEADACQoIAOAEBQQAcIICAgA4QQEBAJyggAAATlBAAAAnfMYY43oRnxcOhxUMBjVT85RgMYzUi9Bt13jKjbmryjozLbXGOrMrPMo6c9DD8MRI1Nv3IYlxUevM4MR268wgD0Muk+I7rTOSFCf7T4eoh2GkQ+Ltj8OQhDbrTEpCq3VGkgLx9rk4n/354EW8h/+jd0KXxn4hZxDw8P/UYew/BwuD1dYZSfp1zVetM8FvHLDavsNEVKYNCoVCSklJOeN2XAEBAJyggAAATlBAAAAnKCAAgBMUEADACQoIAOAEBQQAcIICAgA4QQEBAJyggAAATlBAAAAnKCAAgBN9dxhp3M12w0ij3oZP9pamBQXWmYL737XPBOwHFE5IqrfOSFKi7IdPDvIwsHJInP2wz1aPp7WX78i2teRaZzo97GnLJ1dYZyIehlxKUn3zmQdInkmixwGwtqLG/nxo6fA22DjUMsg6Ex9nf+61lqVbZ4a/Zz+kV5L8b9h/XbHFMFIAQJ9GAQEAnKCAAABOUEAAACcoIACAExQQAMAJCggA4AQFBABwggICADhBAQEAnKCAAABOUEAAACf67jBSzbMbRgrPfFMnecq1ZCVbZ/zH26wzjaPt95NS3WSdkaS4tg7rTPT//tXTvoCBimGkAIA+jQICADhhVUClpaWaOnWqAoGAMjIyNH/+fFVVVXXbZubMmfL5fN1uixcvjumiAQD9n1UBlZeXq6SkRJWVlXrzzTcViUQ0e/ZsNTV1/3n7HXfcodra2q7bqlWrYrpoAED/l2Cz8aZNm7p9vHbtWmVkZGjnzp2aMWNG1/2DBw9WVlZWbFYIABiQLug5oFAoJElKS0vrdv/zzz+v9PR0TZw4UStWrFBzc/MZ/462tjaFw+FuNwDAwGd1BfR50WhUy5Yt0/Tp0zVx4sSu+7/zne9o9OjRysnJ0d69e3XfffepqqpKr7766mn/ntLSUj3yyCNelwEA6Kc8vw9oyZIl+t3vfqdt27Zp5MiRZ9xuy5YtmjVrlg4cOKCxY8ee8nhbW5va2j57b0g4HFZubi7vA+pFvA/oM7wPCLhw5/s+IE9XQEuXLtXGjRu1devWs5aPJBUUFEjSGQvI7/fL7/d7WQYAoB+zKiBjjO6++26tX79eZWVlysvLO2dmz549kqTs7GxPCwQADExWBVRSUqJ169Zpw4YNCgQCqqurkyQFg0ElJyerurpa69at0ze+8Q0NHz5ce/fu1T333KMZM2Zo8uTJPfIPAAD0T1YF9Mwzz0g6+WbTz1uzZo0WLVqkpKQkvfXWW3ryySfV1NSk3NxcLViwQA888EDMFgwAGBisfwR3Nrm5uSovL7+gBQEALg6eX4aNgcO8+2dPuUExXseZpPypl3YkKdp7uwIuegwjBQA4QQEBAJyggAAATlBAAAAnKCAAgBMUEADACQoIAOAEBQQAcIICAgA4QQEBAJyggAAATlBAAAAnKCAAgBMUEADACQoIAOAEBQQAcIICAgA4QQEBAJyggAAATlBAAAAnKCAAgBMUEADACQoIAOAEBQQAcCLB9QK+yBgjSepQRDKOFwMAsNahiKTPvp6fSZ8roMbGRknSNr3heCUAgAvR2NioYDB4xsd95lwV1cui0agOHz6sQCAgn8/X7bFwOKzc3FwdOnRIKSkpjlboHsfhJI7DSRyHkzgOJ/WF42CMUWNjo3JychQXd+ZnevrcFVBcXJxGjhx51m1SUlIu6hPsUxyHkzgOJ3EcTuI4nOT6OJztyudTvAgBAOAEBQQAcKJfFZDf79fKlSvl9/tdL8UpjsNJHIeTOA4ncRxO6k/Hoc+9CAEAcHHoV1dAAICBgwICADhBAQEAnKCAAABOUEAAACf6TQGtXr1al156qQYNGqSCggK98847rpfU6x5++GH5fL5utwkTJrheVo/bunWrbrzxRuXk5Mjn8+m1117r9rgxRg899JCys7OVnJysoqIi7d+/381ie9C5jsOiRYtOOT/mzp3rZrE9pLS0VFOnTlUgEFBGRobmz5+vqqqqbtu0traqpKREw4cP19ChQ7VgwQLV19c7WnHPOJ/jMHPmzFPOh8WLFzta8en1iwJ66aWXtHz5cq1cuVK7du1Sfn6+5syZoyNHjrheWq+76qqrVFtb23Xbtm2b6yX1uKamJuXn52v16tWnfXzVqlX6xS9+oWeffVbbt2/XkCFDNGfOHLW2tvbySnvWuY6DJM2dO7fb+fHCCy/04gp7Xnl5uUpKSlRZWak333xTkUhEs2fPVlNTU9c299xzj15//XW98sorKi8v1+HDh3XzzTc7XHXsnc9xkKQ77rij2/mwatUqRys+A9MPTJs2zZSUlHR93NnZaXJyckxpaanDVfW+lStXmvz8fNfLcEqSWb9+fdfH0WjUZGVlmccff7zrvoaGBuP3+80LL7zgYIW944vHwRhjFi5caObNm+dkPa4cOXLESDLl5eXGmJP/94mJieaVV17p2uavf/2rkWQqKipcLbPHffE4GGPMddddZ37wgx+4W9R56PNXQO3t7dq5c6eKioq67ouLi1NRUZEqKiocrsyN/fv3KycnR2PGjNFtt92mgwcPul6SUzU1Naqrq+t2fgSDQRUUFFyU50dZWZkyMjI0fvx4LVmyRMePH3e9pB4VCoUkSWlpaZKknTt3KhKJdDsfJkyYoFGjRg3o8+GLx+FTzz//vNLT0zVx4kStWLFCzc3NLpZ3Rn1uGvYXHTt2TJ2dncrMzOx2f2Zmpv72t785WpUbBQUFWrt2rcaPH6/a2lo98sgj+trXvqZ9+/YpEAi4Xp4TdXV1knTa8+PTxy4Wc+fO1c0336y8vDxVV1fr/vvvV3FxsSoqKhQfH+96eTEXjUa1bNkyTZ8+XRMnTpR08nxISkpSampqt20H8vlwuuMgSd/5znc0evRo5eTkaO/evbrvvvtUVVWlV1991eFqu+vzBYTPFBcXd/158uTJKigo0OjRo/Xyyy/r9ttvd7gy9AW33npr158nTZqkyZMna+zYsSorK9OsWbMcrqxnlJSUaN++fRfF86Bnc6bjcOedd3b9edKkScrOztasWbNUXV2tsWPH9vYyT6vP/wguPT1d8fHxp7yKpb6+XllZWY5W1TekpqZq3LhxOnDggOulOPPpOcD5caoxY8YoPT19QJ4fS5cu1caNG/WHP/yh2+8Py8rKUnt7uxoaGrptP1DPhzMdh9MpKCiQpD51PvT5AkpKStKUKVO0efPmrvui0ag2b96swsJChytz78SJE6qurlZ2drbrpTiTl5enrKysbudHOBzW9u3bL/rz48MPP9Tx48cH1PlhjNHSpUu1fv16bdmyRXl5ed0enzJlihITE7udD1VVVTp48OCAOh/OdRxOZ8+ePZLUt84H16+COB8vvvii8fv9Zu3atea9994zd955p0lNTTV1dXWul9arfvjDH5qysjJTU1Nj/vjHP5qioiKTnp5ujhw54nppPaqxsdHs3r3b7N6920gyTzzxhNm9e7f54IMPjDHG/PSnPzWpqalmw4YNZu/evWbevHkmLy/PtLS0OF55bJ3tODQ2Npof/ehHpqKiwtTU1Ji33nrLfPnLXzaXX365aW1tdb30mFmyZIkJBoOmrKzM1NbWdt2am5u7tlm8eLEZNWqU2bJli9mxY4cpLCw0hYWFDlcde+c6DgcOHDCPPvqo2bFjh6mpqTEbNmwwY8aMMTNmzHC88u76RQEZY8xTTz1lRo0aZZKSksy0adNMZWWl6yX1ultuucVkZ2ebpKQkc8kll5hbbrnFHDhwwPWyetwf/vAHI+mU28KFC40xJ1+K/eCDD5rMzEzj9/vNrFmzTFVVldtF94CzHYfm5mYze/ZsM2LECJOYmGhGjx5t7rjjjgH3Tdrp/v2SzJo1a7q2aWlpMXfddZcZNmyYGTx4sLnppptMbW2tu0X3gHMdh4MHD5oZM2aYtLQ04/f7zWWXXWb++Z//2YRCIbcL/wJ+HxAAwIk+/xwQAGBgooAAAE5QQAAAJyggAIATFBAAwAkKCADgBAUEAHCCAgIAOEEBAQCcoIAAAE5QQAAAJ/4fSFZm765APLcAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "image, label = train_data[0]\n",
    "print(f\"Image shape: {image.shape}\")\n",
    "plt.imshow(image.squeeze()) # image shape is [1, 28, 28] (colour channels, height, width)\n",
    "plt.title(label);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### We can look at the original grayscale image."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaAAAAGzCAYAAABpdMNsAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy88F64QAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAomElEQVR4nO3de3RV5Z3G8eckJIdAksMl5FYCCTdh5KKDECNyj0C0DBSseFmzoINamdAW0LGLmVbqtGtSsWNZVCq20wXWiSLO4lJdSoeLhCogBWHQGWUIBgFDwqXmJCTkQvLOHyzPeLiFd5vkTcL3s9ZecvZ5f9kvLzt53Dn7/I7PGGMEAEALi3A9AQDAjYkAAgA4QQABAJwggAAAThBAAAAnCCAAgBMEEADACQIIAOAEAQQAcIIAAhoxZ84cxcbGNjpu3LhxGjduXJMdd9y4cRo8eHCTfT2gtSGA0C79+te/ls/nU2ZmpuuptEn/8i//og0bNrieBto5AgjtUn5+vtLT07Vnzx4VFha6nk6bQwChJRBAaHeKioq0c+dOPffcc+rRo4fy8/NdTwnAFRBAaHfy8/PVtWtX3XPPPbr33nuvGEBHjx6Vz+fTL37xC/3mN79R37595ff7NWLECP35z39u9BgHDhxQjx49NG7cOJ07d+6q42pqarRkyRL169dPfr9faWlpevLJJ1VTU3Pdf599+/bpjjvuUExMjDIyMrRy5crLxpw6dUpz585VUlKSOnbsqGHDhumll166bFxlZaUef/xxpaWlye/366abbtIvfvELfbUpvs/nU2VlpV566SX5fD75fD7NmTPnuucLXDcDtDMDBw40c+fONcYYs2PHDiPJ7NmzJ2xMUVGRkWRuvfVW069fP/PMM8+YpUuXmoSEBNOzZ09TW1sbGjt79mzTuXPn0OM9e/aYrl27mrvuustUVVWF9o8dO9aMHTs29Li+vt5MmjTJdOrUySxYsMC8+OKLZv78+aZDhw5m2rRpjf49xo4da1JTU01iYqKZP3++Wb58ubnzzjuNJPO73/0uNK6qqsoMGjTIREVFmYULF5rly5eb0aNHG0lm2bJloXENDQ1mwoQJxufzmYcfftg8//zzZurUqUaSWbBgQWjcyy+/bPx+vxk9erR5+eWXzcsvv2x27tzZ+MIDlgggtCt79+41kszmzZuNMRd/6Pbs2dP84Ac/CBv3ZQB1797d/OUvfwnt37hxo5Fk3njjjdC+rwbQu+++a+Lj480999xjqqurw77mpQH08ssvm4iICPOnP/0pbNzKlSuNJPPee+9d8+8yduxYI8n867/+a2hfTU2NueWWW0xiYmIoJJctW2YkmX//938PjautrTVZWVkmNjbWlJeXG2OM2bBhg5Fkfvazn4Ud59577zU+n88UFhaG9nXu3NnMnj37mvMDvi5+BYd2JT8/X0lJSRo/fryki79OmjVrltasWaP6+vrLxs+aNUtdu3YNPR49erQk6dNPP71s7DvvvKPJkydr4sSJWrdunfx+/zXn8vrrr2vQoEEaOHCgzpw5E9omTJgQ+nqN6dChg7773e+GHkdHR+u73/2uTp06pX379kmS3nrrLSUnJ+uBBx4IjYuKitL3v/99nTt3TgUFBaFxkZGR+v73vx92jMcff1zGGL399tuNzgdoSgQQ2o36+nqtWbNG48ePV1FRkQoLC1VYWKjMzEyVlpZq69atl9X06tUr7PGXYfTFF1+E7a+urtY999yjW2+9VWvXrlV0dHSj8zl8+LD++7//Wz169AjbBgwYIOni6zaNSU1NVefOncP2fVl/9OhRSdJnn32m/v37KyIi/Nt50KBBoee//G9qaqri4uKuOQ5oKR1cTwBoKtu2bdPJkye1Zs0arVmz5rLn8/PzNWnSpLB9kZGRV/xa5pJPqvf7/br77ru1ceNGbdq0Sd/85jcbnU9DQ4OGDBmi55577orPp6WlNfo1gPaMAEK7kZ+fr8TERK1YseKy59atW6f169dr5cqViomJsf7aPp9P+fn5mjZtmr797W/r7bffbrTrQd++ffVf//Vfmjhxonw+n/UxJam4uFiVlZVhV0H/+7//K0lKT0+XJPXu3VsHDx5UQ0ND2FXQJ598Enr+y/9u2bJFFRUVYVdBl4778u8LNDd+BYd24fz581q3bp2++c1v6t57771smz9/vioqKvSHP/zB8zGio6O1bt06jRgxQlOnTtWePXuuOf6+++7T559/rt/+9rdXnG9lZWWjx7xw4YJefPHF0OPa2lq9+OKL6tGjh4YPHy5Juvvuu1VSUqLXXnstrO5Xv/qVYmNjNXbs2NC4+vp6Pf/882HH+OUvfymfz6ecnJzQvs6dO6usrKzR+QFfB1dAaBf+8Ic/qKKiQn/zN39zxedvv/320JtSZ82a5fk4MTExevPNNzVhwgTl5OSooKDgqv3a/vZv/1Zr167VY489pnfeeUejRo1SfX29PvnkE61du1Z//OMfddttt13zeKmpqXrmmWd09OhRDRgwQK+99poOHDig3/zmN4qKipIkPfroo3rxxRc1Z84c7du3T+np6fqP//gPvffee1q2bFnoamfq1KkaP368/umf/klHjx7VsGHD9J//+Z/auHGjFixYoL59+4aOO3z4cG3ZskXPPfecUlNTlZGRQVsjND3Xt+EBTWHq1KmmY8eOprKy8qpj5syZY6KiosyZM2dCt2E/++yzl42TZJYsWRJ6fOn7gIwx5syZM+av/uqvTHJysjl8+LAx5vLbsI25eDv0M888Y26++Wbj9/tN165dzfDhw83TTz9tgsHgNf9OY8eONTfffLPZu3evycrKMh07djS9e/c2zz///GVjS0tLzXe+8x2TkJBgoqOjzZAhQ8yqVasuG1dRUWEWLlxoUlNTTVRUlOnfv7959tlnTUNDQ9i4Tz75xIwZM8bExMQYSdySjWbhM+aSV1sBAGgBvAYEAHCCAAIAOEEAAQCcIIAAAE4QQAAAJwggAIATre6NqA0NDSouLlZcXBztQACgDTLGqKKiQqmpqZc1yf2qVhdAxcXFNGkEgHbg+PHj6tmz51Wfb3W/gru0VTwAoG1q7Od5swXQihUrlJ6ero4dOyozM7PRxo1f4tduANA+NPbzvFkC6LXXXtOiRYu0ZMkSffDBBxo2bJgmT558XR/ABQC4QTRHg7mRI0ea3Nzc0OP6+nqTmppq8vLyGq0NBoNGEhsbGxtbG98aa7jb5FdAtbW12rdvn7Kzs0P7IiIilJ2drV27dl02vqamRuXl5WEbAKD9a/IAOnPmjOrr65WUlBS2PykpSSUlJZeNz8vLUyAQCG3cAQcANwbnd8EtXrxYwWAwtB0/ftz1lAAALaDJ3weUkJCgyMhIlZaWhu0vLS1VcnLyZeP9fr/8fn9TTwMA0Mo1+RVQdHS0hg8frq1bt4b2NTQ0aOvWrcrKymrqwwEA2qhm6YSwaNEizZ49W7fddptGjhypZcuWqbKyUt/5znea43AAgDaoWQJo1qxZOn36tJ566imVlJTolltu0aZNmy67MQEAcOPyGWOM60l8VXl5uQKBgOtpAAC+pmAwqPj4+Ks+7/wuOADAjYkAAgA4QQABAJwggAAAThBAAAAnCCAAgBMEEADACQIIAOAEAQQAcIIAAgA4QQABAJwggAAAThBAAAAnCCAAgBMEEADACQIIAOAEAQQAcIIAAgA4QQABAJwggAAAThBAAAAnCCAAgBMEEADACQIIAOAEAQQAcIIAAgA40cH1BIDWxOfzWdcYY5phJpeLi4uzrrnzzjs9Hevtt9/2VGfLy3pHRkZa11y4cMG6prXzsnZeNdc5zhUQAMAJAggA4AQBBABwggACADhBAAEAnCCAAABOEEAAACcIIACAEwQQAMAJAggA4AQBBABwggACADhBM1LgKyIi7P+frL6+3rqmX79+1jUPP/ywdc358+etaySpsrLSuqa6utq6Zs+ePdY1LdlY1EvDTy/nkJfjtOQ62DaANcaooaGh0XFcAQEAnCCAAABOEEAAACcIIACAEwQQAMAJAggA4AQBBABwggACADhBAAEAnCCAAABOEEAAACcIIACAEzQjBb7Ctumi5K0Z6YQJE6xrsrOzrWtOnDhhXSNJfr/fuqZTp07WNXfddZd1zb/9279Z15SWllrXSBebatrycj54ERsb66nuepqEXqqqqsrTsRrDFRAAwAkCCADgRJMH0E9+8hP5fL6wbeDAgU19GABAG9csrwHdfPPN2rJly/8fpAMvNQEAwjVLMnTo0EHJycnN8aUBAO1Es7wGdPjwYaWmpqpPnz566KGHdOzYsauOrampUXl5edgGAGj/mjyAMjMztXr1am3atEkvvPCCioqKNHr0aFVUVFxxfF5engKBQGhLS0tr6ikBAFqhJg+gnJwcffvb39bQoUM1efJkvfXWWyorK9PatWuvOH7x4sUKBoOh7fjx4009JQBAK9Tsdwd06dJFAwYMUGFh4RWf9/v9nt70BgBo25r9fUDnzp3TkSNHlJKS0tyHAgC0IU0eQE888YQKCgp09OhR7dy5U9/61rcUGRmpBx54oKkPBQBow5r8V3AnTpzQAw88oLNnz6pHjx668847tXv3bvXo0aOpDwUAaMOaPIDWrFnT1F8SaDG1tbUtcpwRI0ZY16Snp1vXeGmuKkkREfa/HPnjH/9oXXPrrbda1yxdutS6Zu/evdY1kvThhx9a13z88cfWNSNHjrSu8XIOSdLOnTuta3bt2mU13hhzXW+poRccAMAJAggA4AQBBABwggACADhBAAEAnCCAAABOEEAAACcIIACAEwQQAMAJAggA4AQBBABwggACADjR7B9IB7jg8/k81RljrGvuuusu65rbbrvNuuZqH2t/LZ07d7aukaQBAwa0SM2f//xn65qrfbjltcTGxlrXSFJWVpZ1zYwZM6xr6urqrGu8rJ0kPfzww9Y1NTU1VuMvXLigP/3pT42O4woIAOAEAQQAcIIAAgA4QQABAJwggAAAThBAAAAnCCAAgBMEEADACQIIAOAEAQQAcIIAAgA4QQABAJwggAAATviMl/a/zai8vFyBQMD1NNBMvHapbilevh12795tXZOenm5d44XX9b5w4YJ1TW1tradj2aqurrauaWho8HSsDz74wLrGS7duL+s9ZcoU6xpJ6tOnj3XNN77xDU/HCgaDio+Pv+rzXAEBAJwggAAAThBAAAAnCCAAgBMEEADACQIIAOAEAQQAcIIAAgA4QQABAJwggAAAThBAAAAnCCAAgBMdXE8AN5ZW1vu2SXzxxRfWNSkpKdY158+ft67x+/3WNZLUoYP9j4bY2FjrGi+NRWNiYqxrvDYjHT16tHXNHXfcYV0TEWF/LZCYmGhdI0mbNm3yVNccuAICADhBAAEAnCCAAABOEEAAACcIIACAEwQQAMAJAggA4AQBBABwggACADhBAAEAnCCAAABOEEAAACdoRgp8TZ06dbKu8dJ80ktNVVWVdY0kBYNB65qzZ89a16Snp1vXeGlo6/P5rGskb2vu5Xyor6+3rvHaYDUtLc1TXXPgCggA4AQBBABwwjqAduzYoalTpyo1NVU+n08bNmwIe94Yo6eeekopKSmKiYlRdna2Dh8+3FTzBQC0E9YBVFlZqWHDhmnFihVXfH7p0qVavny5Vq5cqffff1+dO3fW5MmTPX3wFACg/bK+CSEnJ0c5OTlXfM4Yo2XLlulHP/qRpk2bJkn6/e9/r6SkJG3YsEH333//15stAKDdaNLXgIqKilRSUqLs7OzQvkAgoMzMTO3ateuKNTU1NSovLw/bAADtX5MGUElJiSQpKSkpbH9SUlLouUvl5eUpEAiEttZ0iyAAoPk4vwtu8eLFCgaDoe348eOupwQAaAFNGkDJycmSpNLS0rD9paWloecu5ff7FR8fH7YBANq/Jg2gjIwMJScna+vWraF95eXlev/995WVldWUhwIAtHHWd8GdO3dOhYWFocdFRUU6cOCAunXrpl69emnBggX62c9+pv79+ysjI0M//vGPlZqaqunTpzflvAEAbZx1AO3du1fjx48PPV60aJEkafbs2Vq9erWefPJJVVZW6tFHH1VZWZnuvPNObdq0SR07dmy6WQMA2jyf8dLZrxmVl5crEAi4ngaaiZemkF4aQnpp7ihJsbGx1jX79++3rvGyDufPn7eu8fv91jWSVFxcbF1z6Wu/1+OOO+6wrvHS9NRLg1BJio6Otq6pqKiwrvHyM8/rDVtezvG5c+daja+vr9f+/fsVDAav+bq+87vgAAA3JgIIAOAEAQQAcIIAAgA4QQABAJwggAAAThBAAAAnCCAAgBMEEADACQIIAOAEAQQAcIIAAgA4QQABAJyw/jgG4Ovw0nw9MjLSusZrN+xZs2ZZ11zt036v5fTp09Y1MTEx1jUNDQ3WNZLUuXNn65q0tDTrmtraWusaLx2+6+rqrGskqUMH+x+RXv6dunfvbl2zYsUK6xpJuuWWW6xrvKzD9eAKCADgBAEEAHCCAAIAOEEAAQCcIIAAAE4QQAAAJwggAIATBBAAwAkCCADgBAEEAHCCAAIAOEEAAQCcoBkpWpSXpoZeGlZ69dFHH1nX1NTUWNdERUVZ17RkU9bExETrmurqauuas2fPWtd4WbuOHTta10jemrJ+8cUX1jUnTpywrnnwwQetayTp2Wefta7ZvXu3p2M1hisgAIATBBAAwAkCCADgBAEEAHCCAAIAOEEAAQCcIIAAAE4QQAAAJwggAIATBBAAwAkCCADgBAEEAHDihm5G6vP5PNV5aQoZEWGf9V7mV1dXZ13T0NBgXePVhQsXWuxYXrz11lvWNZWVldY158+ft66Jjo62rjHGWNdI0unTp61rvHxfeGkS6uUc96qlvp+8rN3QoUOtayQpGAx6qmsOXAEBAJwggAAAThBAAAAnCCAAgBMEEADACQIIAOAEAQQAcIIAAgA4QQABAJwggAAAThBAAAAnCCAAgBPtphmpl2Z+9fX1no7V2htqtmZjxoyxrpk5c6Z1zahRo6xrJKmqqsq65uzZs9Y1XhqLduhg/+3q9Rz3sg5evgf9fr91jZcGpl6bsnpZBy+8nA/nzp3zdKwZM2ZY17zxxhuejtUYroAAAE4QQAAAJ6wDaMeOHZo6dapSU1Pl8/m0YcOGsOfnzJkjn88Xtk2ZMqWp5gsAaCesA6iyslLDhg3TihUrrjpmypQpOnnyZGh79dVXv9YkAQDtj/Wrmjk5OcrJybnmGL/fr+TkZM+TAgC0f83yGtD27duVmJiom266SfPmzbvmXUI1NTUqLy8P2wAA7V+TB9CUKVP0+9//Xlu3btUzzzyjgoIC5eTkXPV20Ly8PAUCgdCWlpbW1FMCALRCTf4+oPvvvz/05yFDhmjo0KHq27evtm/frokTJ142fvHixVq0aFHocXl5OSEEADeAZr8Nu0+fPkpISFBhYeEVn/f7/YqPjw/bAADtX7MH0IkTJ3T27FmlpKQ096EAAG2I9a/gzp07F3Y1U1RUpAMHDqhbt27q1q2bnn76ac2cOVPJyck6cuSInnzySfXr10+TJ09u0okDANo26wDau3evxo8fH3r85es3s2fP1gsvvKCDBw/qpZdeUllZmVJTUzVp0iT99Kc/9dTzCQDQfvmM1y59zaS8vFyBQMD1NJpct27drGtSU1Ota/r3798ix5G8NTUcMGCAdU1NTY11TUSEt98u19XVWdfExMRY1xQXF1vXREVFWdd4aXIpSd27d7euqa2tta7p1KmTdc3OnTuta2JjY61rJG/NcxsaGqxrgsGgdY2X80GSSktLrWsGDRrk6VjBYPCar+vTCw4A4AQBBABwggACADhBAAEAnCCAAABOEEAAACcIIACAEwQQAMAJAggA4AQBBABwggACADhBAAEAnCCAAABONPlHcrty++23W9f89Kc/9XSsHj16WNd06dLFuqa+vt66JjIy0rqmrKzMukaSLly4YF1TUVFhXeOly7LP57OukaTz589b13jpznzfffdZ1+zdu9e6Ji4uzrpG8taBPD093dOxbA0ZMsS6xus6HD9+3LqmqqrKusZLR3WvHb579+7tqa45cAUEAHCCAAIAOEEAAQCcIIAAAE4QQAAAJwggAIATBBAAwAkCCADgBAEEAHCCAAIAOEEAAQCcIIAAAE602makERERVg0lly9fbn2MlJQU6xrJW5NQLzVemhp6ER0d7anOy9/JS7NPLwKBgKc6L40af/7zn1vXeFmHefPmWdcUFxdb10hSdXW1dc3WrVutaz799FPrmv79+1vXdO/e3bpG8tYINyoqyromIsL+WqCurs66RpJOnz7tqa45cAUEAHCCAAIAOEEAAQCcIIAAAE4QQAAAJwggAIATBBAAwAkCCADgBAEEAHCCAAIAOEEAAQCcIIAAAE74jDHG9SS+qry8XIFAQA899JBVk0wvDSGPHDliXSNJsbGxLVLj9/uta7zw0jxR8tbw8/jx49Y1Xhpq9ujRw7pG8tYUMjk52bpm+vTp1jUdO3a0rklPT7eukbydr8OHD2+RGi//Rl6aino9ltfmvrZsmjV/lZfv99tvv91qfENDgz7//HMFg0HFx8dfdRxXQAAAJwggAIATBBAAwAkCCADgBAEEAHCCAAIAOEEAAQCcIIAAAE4QQAAAJwggAIATBBAAwAkCCADgRAfXE7ia06dPWzXN89LkMi4uzrpGkmpqaqxrvMzPS0NIL40Qr9Us8Fr+8pe/WNd89tln1jVe1uH8+fPWNZJUXV1tXXPhwgXrmvXr11vXfPjhh9Y1XpuRduvWzbrGS8PPsrIy65q6ujrrGi//RtLFppq2vDT79HIcr81IvfyMGDBggNX4Cxcu6PPPP290HFdAAAAnCCAAgBNWAZSXl6cRI0YoLi5OiYmJmj59ug4dOhQ2prq6Wrm5uerevbtiY2M1c+ZMlZaWNumkAQBtn1UAFRQUKDc3V7t379bmzZtVV1enSZMmqbKyMjRm4cKFeuONN/T666+roKBAxcXFmjFjRpNPHADQtlndhLBp06awx6tXr1ZiYqL27dunMWPGKBgM6ne/+51eeeUVTZgwQZK0atUqDRo0SLt377b+VD0AQPv1tV4DCgaDkv7/jpl9+/aprq5O2dnZoTEDBw5Ur169tGvXrit+jZqaGpWXl4dtAID2z3MANTQ0aMGCBRo1apQGDx4sSSopKVF0dLS6dOkSNjYpKUklJSVX/Dp5eXkKBAKhLS0tzeuUAABtiOcAys3N1UcffaQ1a9Z8rQksXrxYwWAwtHl5vwwAoO3x9EbU+fPn680339SOHTvUs2fP0P7k5GTV1taqrKws7CqotLRUycnJV/xafr9ffr/fyzQAAG2Y1RWQMUbz58/X+vXrtW3bNmVkZIQ9P3z4cEVFRWnr1q2hfYcOHdKxY8eUlZXVNDMGALQLVldAubm5euWVV7Rx40bFxcWFXtcJBAKKiYlRIBDQ3LlztWjRInXr1k3x8fH63ve+p6ysLO6AAwCEsQqgF154QZI0bty4sP2rVq3SnDlzJEm//OUvFRERoZkzZ6qmpkaTJ0/Wr3/96yaZLACg/fAZY4zrSXxVeXm5AoGAhgwZosjIyOuu++1vf2t9rDNnzljXSFLnzp2ta7p3725d46VR47lz56xrvDRPlKQOHexfQvTSdLFTp07WNV4amEre1iIiwv5eHi/fdpfeXXo9vvomcRtemrl+8cUX1jVeXv/18n3rpYGp5K2JqZdjxcTEWNdc7XX1xnhpYpqfn281vqamRs8//7yCweA1mx3TCw4A4AQBBABwggACADhBAAEAnCCAAABOEEAAACcIIACAEwQQAMAJAggA4AQBBABwggACADhBAAEAnCCAAABOePpE1Jbw4YcfWo1ft26d9TH+7u/+zrpGkoqLi61rPv30U+ua6upq6xovXaC9dsP20sE3OjrausamK/qXampqrGskqb6+3rrGS2frqqoq65qTJ09a13htdu9lHbx0R2+pc7y2tta6RvLWkd5LjZcO2l46dUu67INEr0dpaanV+Otdb66AAABOEEAAACcIIACAEwQQAMAJAggA4AQBBABwggACADhBAAEAnCCAAABOEEAAACcIIACAEwQQAMAJn/HarbCZlJeXKxAItMixcnJyPNU98cQT1jWJiYnWNWfOnLGu8dII0UvjSclbk1AvzUi9NLn0MjdJ8vl81jVevoW8NID1UuNlvb0ey8vaeeHlOLbNNL8OL2ve0NBgXZOcnGxdI0kHDx60rrnvvvs8HSsYDCo+Pv6qz3MFBABwggACADhBAAEAnCCAAABOEEAAACcIIACAEwQQAMAJAggA4AQBBABwggACADhBAAEAnCCAAABOtNpmpD6fz6rpoJdmfi1p/Pjx1jV5eXnWNV6annpt/hoRYf//L16ahHppRuq1waoXp06dsq7x8m33+eefW9d4/b44d+6cdY3XBrC2vKxdXV2dp2NVVVVZ13j5vti8ebN1zccff2xdI0k7d+70VOcFzUgBAK0SAQQAcIIAAgA4QQABAJwggAAAThBAAAAnCCAAgBMEEADACQIIAOAEAQQAcIIAAgA4QQABAJxotc1I0XIGDhzoqS4hIcG6pqyszLqmZ8+e1jVHjx61rpG8Na08cuSIp2MB7R3NSAEArRIBBABwwiqA8vLyNGLECMXFxSkxMVHTp0/XoUOHwsaMGzcu9Fk+X26PPfZYk04aAND2WQVQQUGBcnNztXv3bm3evFl1dXWaNGmSKisrw8Y98sgjOnnyZGhbunRpk04aAND2WX3U5KZNm8Ier169WomJidq3b5/GjBkT2t+pUyclJyc3zQwBAO3S13oNKBgMSpK6desWtj8/P18JCQkaPHiwFi9efM2Pta2pqVF5eXnYBgBo/6yugL6qoaFBCxYs0KhRozR48ODQ/gcffFC9e/dWamqqDh48qB/+8Ic6dOiQ1q1bd8Wvk5eXp6efftrrNAAAbZTn9wHNmzdPb7/9tt59991rvk9j27ZtmjhxogoLC9W3b9/Lnq+pqVFNTU3ocXl5udLS0rxMCR7xPqD/x/uAgKbT2PuAPF0BzZ8/X2+++aZ27NjR6A+HzMxMSbpqAPn9fvn9fi/TAAC0YVYBZIzR9773Pa1fv17bt29XRkZGozUHDhyQJKWkpHiaIACgfbIKoNzcXL3yyivauHGj4uLiVFJSIkkKBAKKiYnRkSNH9Morr+juu+9W9+7ddfDgQS1cuFBjxozR0KFDm+UvAABom6wC6IUXXpB08c2mX7Vq1SrNmTNH0dHR2rJli5YtW6bKykqlpaVp5syZ+tGPftRkEwYAtA/Wv4K7lrS0NBUUFHytCQEAbgx0wwYANAu6YQMAWiUCCADgBAEEAHCCAAIAOEEAAQCcIIAAAE4QQAAAJwggAIATBBAAwAkCCADgBAEEAHCCAAIAOEEAAQCcIIAAAE4QQAAAJwggAIATBBAAwAkCCADgBAEEAHCCAAIAOEEAAQCcIIAAAE4QQAAAJwggAIATrS6AjDGupwAAaAKN/TxvdQFUUVHhegoAgCbQ2M9zn2lllxwNDQ0qLi5WXFycfD5f2HPl5eVKS0vT8ePHFR8f72iG7rEOF7EOF7EOF7EOF7WGdTDGqKKiQqmpqYqIuPp1TocWnNN1iYiIUM+ePa85Jj4+/oY+wb7EOlzEOlzEOlzEOlzkeh0CgUCjY1rdr+AAADcGAggA4ESbCiC/368lS5bI7/e7nopTrMNFrMNFrMNFrMNFbWkdWt1NCACAG0ObugICALQfBBAAwAkCCADgBAEEAHCCAAIAONFmAmjFihVKT09Xx44dlZmZqT179rieUov7yU9+Ip/PF7YNHDjQ9bSa3Y4dOzR16lSlpqbK5/Npw4YNYc8bY/TUU08pJSVFMTExys7O1uHDh91Mthk1tg5z5sy57PyYMmWKm8k2k7y8PI0YMUJxcXFKTEzU9OnTdejQobAx1dXVys3NVffu3RUbG6uZM2eqtLTU0Yybx/Wsw7hx4y47Hx577DFHM76yNhFAr732mhYtWqQlS5bogw8+0LBhwzR58mSdOnXK9dRa3M0336yTJ0+Gtnfffdf1lJpdZWWlhg0bphUrVlzx+aVLl2r58uVauXKl3n//fXXu3FmTJ09WdXV1C8+0eTW2DpI0ZcqUsPPj1VdfbcEZNr+CggLl5uZq9+7d2rx5s+rq6jRp0iRVVlaGxixcuFBvvPGGXn/9dRUUFKi4uFgzZsxwOOumdz3rIEmPPPJI2PmwdOlSRzO+CtMGjBw50uTm5oYe19fXm9TUVJOXl+dwVi1vyZIlZtiwYa6n4ZQks379+tDjhoYGk5ycbJ599tnQvrKyMuP3+82rr77qYIYt49J1MMaY2bNnm2nTpjmZjyunTp0ykkxBQYEx5uK/fVRUlHn99ddDYz7++GMjyezatcvVNJvdpetgjDFjx441P/jBD9xN6jq0+iug2tpa7du3T9nZ2aF9ERERys7O1q5duxzOzI3Dhw8rNTVVffr00UMPPaRjx465npJTRUVFKikpCTs/AoGAMjMzb8jzY/v27UpMTNRNN92kefPm6ezZs66n1KyCwaAkqVu3bpKkffv2qa6uLux8GDhwoHr16tWuz4dL1+FL+fn5SkhI0ODBg7V48WJVVVW5mN5Vtbpu2Jc6c+aM6uvrlZSUFLY/KSlJn3zyiaNZuZGZmanVq1frpptu0smTJ/X0009r9OjR+uijjxQXF+d6ek6UlJRI0hXPjy+fu1FMmTJFM2bMUEZGho4cOaJ//Md/VE5Ojnbt2qXIyEjX02tyDQ0NWrBggUaNGqXBgwdLung+REdHq0uXLmFj2/P5cKV1kKQHH3xQvXv3Vmpqqg4ePKgf/vCHOnTokNatW+dwtuFafQDh/+Xk5IT+PHToUGVmZqp3795au3at5s6d63BmaA3uv//+0J+HDBmioUOHqm/fvtq+fbsmTpzocGbNIzc3Vx999NEN8TrotVxtHR599NHQn4cMGaKUlBRNnDhRR44cUd++fVt6mlfU6n8Fl5CQoMjIyMvuYiktLVVycrKjWbUOXbp00YABA1RYWOh6Ks58eQ5wflyuT58+SkhIaJfnx/z58/Xmm2/qnXfeCfv8sOTkZNXW1qqsrCxsfHs9H662DleSmZkpSa3qfGj1ARQdHa3hw4dr69atoX0NDQ3aunWrsrKyHM7MvXPnzunIkSNKSUlxPRVnMjIylJycHHZ+lJeX6/3337/hz48TJ07o7Nmz7er8MMZo/vz5Wr9+vbZt26aMjIyw54cPH66oqKiw8+HQoUM6duxYuzofGluHKzlw4IAkta7zwfVdENdjzZo1xu/3m9WrV5v/+Z//MY8++qjp0qWLKSkpcT21FvX444+b7du3m6KiIvPee++Z7Oxsk5CQYE6dOuV6as2qoqLC7N+/3+zfv99IMs8995zZv3+/+eyzz4wxxvz85z83Xbp0MRs3bjQHDx4006ZNMxkZGeb8+fOOZ960rrUOFRUV5oknnjC7du0yRUVFZsuWLeav//qvTf/+/U11dbXrqTeZefPmmUAgYLZv325OnjwZ2qqqqkJjHnvsMdOrVy+zbds2s3fvXpOVlWWysrIczrrpNbYOhYWF5p//+Z/N3r17TVFRkdm4caPp06ePGTNmjOOZh2sTAWSMMb/61a9Mr169THR0tBk5cqTZvXu36ym1uFmzZpmUlBQTHR1tvvGNb5hZs2aZwsJC19Nqdu+8846RdNk2e/ZsY8zFW7F//OMfm6SkJOP3+83EiRPNoUOH3E66GVxrHaqqqsykSZNMjx49TFRUlOndu7d55JFH2t3/pF3p7y/JrFq1KjTm/Pnz5u///u9N165dTadOncy3vvUtc/LkSXeTbgaNrcOxY8fMmDFjTLdu3Yzf7zf9+vUz//AP/2CCwaDbiV+CzwMCADjR6l8DAgC0TwQQAMAJAggA4AQBBABwggACADhBAAEAnCCAAABOEEAAACcIIACAEwQQAMAJAggA4MT/AcBjvi3QnOhnAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(image.squeeze(), cmap=\"gray\")\n",
    "plt.title(class_names[label]);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Now that we know what our targets look like, let's prepare our data."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Prepare data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Here, we will be using **torch.utils.data.DataLoader**.\n",
    "##### In short, this function is what helps us to load data into a model.\n",
    "##### It turns a large dataset into a python iterable of smaller chunks, whic are referred to as batches or mini-batches.\n",
    "##### This is more efficient computationally as passing through all data at once won't be welcomed by your computer.\n",
    "##### Generally, 32 is a good batch size for most problems.\n",
    "##### Usually the batch size can be any power of 2 but this depends on the user."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import DataLoader\n",
    "\n",
    "BATCH_SIZE = 32\n",
    "\n",
    "train_dataloader = DataLoader(train_data, batch_size = BATCH_SIZE, shuffle=True)\n",
    "\n",
    "test_dataloader = DataLoader(test_data, batch_size=BATCH_SIZE, shuffle=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Building a baseline model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### A baseline model is the simplest model we can build.\n",
    "##### For now, it will just have two nn.Linear() layers.\n",
    "##### But since we are working with image data, we're going to be using a different layer to start things off.\n",
    "##### We would have the use the nn.Flatten() layer.\n",
    "##### This layer would be in charge of compressing the dimensions of a tensor into a single vector."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape before flattening: torch.Size([1, 28, 28]) -> [color_channels, height, width]\n",
      "Shape after flattening: torch.Size([1, 784]) -> [color_channels, height*width]\n"
     ]
    }
   ],
   "source": [
    "flatten_model = nn.Flatten()\n",
    "\n",
    "train_features_batch, train_labels_batch = next(iter(train_dataloader))\n",
    "x = train_features_batch[0]\n",
    "\n",
    "output = flatten_model(x)\n",
    "\n",
    "print(f\"Shape before flattening: {x.shape} -> [color_channels, height, width]\")\n",
    "print(f\"Shape after flattening: {output.shape} -> [color_channels, height*width]\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### This has to be performed to change our height and width dimensions into one long feature vector. \n",
    "##### So now, there would be 784 feature variables in total."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FashionMNISTV0(nn.Module):\n",
    "    def __init__(self, input_shape:int, hidden_units:int, output_shape:int):\n",
    "        super().__init__()\n",
    "        self.layer_stack = nn.Sequential(\n",
    "            nn.Flatten(),\n",
    "            nn.Linear(input_shape, hidden_units),\n",
    "            nn.Linear(hidden_units, output_shape)\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        return self.layer_stack(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "FashionMNISTV0(\n",
       "  (layer_stack): Sequential(\n",
       "    (0): Flatten(start_dim=1, end_dim=-1)\n",
       "    (1): Linear(in_features=784, out_features=10, bias=True)\n",
       "    (2): Linear(in_features=10, out_features=10, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.manual_seed(42)\n",
    "\n",
    "model_0 = FashionMNISTV0(784, 10, len(class_names))\n",
    "\n",
    "model_0.to(\"cpu\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Let's bring in some helper function."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "helper_functions.py already exists, skipping download\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "from pathlib import Path \n",
    "\n",
    "# Download helper functions from Learn PyTorch repo (if not already downloaded)\n",
    "if Path(\"helper_functions.py\").is_file():\n",
    "  print(\"helper_functions.py already exists, skipping download\")\n",
    "else:\n",
    "  print(\"Downloading helper_functions.py\")\n",
    "  # Note: you need the \"raw\" GitHub URL for this to work\n",
    "  request = requests.get(\"https://raw.githubusercontent.com/mrdbourke/pytorch-deep-learning/main/helper_functions.py\")\n",
    "  with open(\"helper_functions.py\", \"wb\") as f:\n",
    "    f.write(request.content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import accuracy metric\n",
    "from helper_functions import accuracy_fn # Note: could also use torchmetrics.Accuracy()\n",
    "\n",
    "# Setup loss function and optimizer\n",
    "loss_fn = nn.CrossEntropyLoss() # this is also called \"criterion\"/\"cost function\" in some places\n",
    "optimizer = torch.optim.SGD(params=model_0.parameters(), lr=0.1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Let's implement our training loop now."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/kimhyunbin/Documents/Python/My own project (Python)/PyTorch/pytorch/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "  0%|          | 0/3 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0\n",
      "-------\n",
      "Looked at 0/60000 samples\n",
      "Looked at 12800/60000 samples\n",
      "Looked at 25600/60000 samples\n",
      "Looked at 38400/60000 samples\n",
      "Looked at 51200/60000 samples\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 33%|███▎      | 1/3 [00:01<00:03,  1.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Train loss: 0.59039 | Test loss: 0.50954, Test acc: 82.04%\n",
      "\n",
      "Epoch: 1\n",
      "-------\n",
      "Looked at 0/60000 samples\n",
      "Looked at 12800/60000 samples\n",
      "Looked at 25600/60000 samples\n",
      "Looked at 38400/60000 samples\n",
      "Looked at 51200/60000 samples\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 67%|██████▋   | 2/3 [00:03<00:01,  1.89s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Train loss: 0.47633 | Test loss: 0.47989, Test acc: 83.20%\n",
      "\n",
      "Epoch: 2\n",
      "-------\n",
      "Looked at 0/60000 samples\n",
      "Looked at 12800/60000 samples\n",
      "Looked at 25600/60000 samples\n",
      "Looked at 38400/60000 samples\n",
      "Looked at 51200/60000 samples\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 3/3 [00:05<00:00,  1.89s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Train loss: 0.45503 | Test loss: 0.47664, Test acc: 83.43%\n",
      "\n",
      "Train time on cpu: 5.676 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from timeit import default_timer as timer \n",
    "def print_train_time(start: float, end: float, device: torch.device = None):\n",
    "    total_time = end - start\n",
    "    print(f\"Train time on {device}: {total_time:.3f} seconds\")\n",
    "    return total_time\n",
    "\n",
    "# Import tqdm for progress bar\n",
    "from tqdm.auto import tqdm\n",
    "\n",
    "# Set the seed and start the timer\n",
    "torch.manual_seed(42)\n",
    "train_time_start_on_cpu = timer()\n",
    "\n",
    "# Set the number of epochs (we'll keep this small for faster training times)\n",
    "epochs = 3\n",
    "\n",
    "# Create training and testing loop\n",
    "for epoch in tqdm(range(epochs)):\n",
    "    print(f\"Epoch: {epoch}\\n-------\")\n",
    "    ### Training\n",
    "    train_loss = 0\n",
    "    # Add a loop to loop through training batches\n",
    "    for batch, (X, y) in enumerate(train_dataloader):\n",
    "        model_0.train() \n",
    "        # 1. Forward pass\n",
    "        y_pred = model_0(X)\n",
    "\n",
    "        # 2. Calculate loss (per batch)\n",
    "        loss = loss_fn(y_pred, y)\n",
    "        train_loss += loss # accumulatively add up the loss per epoch \n",
    "\n",
    "        # 3. Optimizer zero grad\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        # 4. Loss backward\n",
    "        loss.backward()\n",
    "\n",
    "        # 5. Optimizer step\n",
    "        optimizer.step()\n",
    "\n",
    "        # Print out how many samples have been seen\n",
    "        if batch % 400 == 0:\n",
    "            print(f\"Looked at {batch * len(X)}/{len(train_dataloader.dataset)} samples\")\n",
    "\n",
    "    # Divide total train loss by length of train dataloader (average loss per batch per epoch)\n",
    "    train_loss /= len(train_dataloader)\n",
    "    \n",
    "    ### Testing\n",
    "    # Setup variables for accumulatively adding up loss and accuracy \n",
    "    test_loss, test_acc = 0, 0 \n",
    "    model_0.eval()\n",
    "    with torch.inference_mode():\n",
    "        for X, y in test_dataloader:\n",
    "            # 1. Forward pass\n",
    "            test_pred = model_0(X)\n",
    "           \n",
    "            # 2. Calculate loss (accumatively)\n",
    "            test_loss += loss_fn(test_pred, y) # accumulatively add up the loss per epoch\n",
    "\n",
    "            # 3. Calculate accuracy (preds need to be same as y_true)\n",
    "            test_acc += accuracy_fn(y_true=y, y_pred=test_pred.argmax(dim=1))\n",
    "        \n",
    "        # Calculations on test metrics need to happen inside torch.inference_mode()\n",
    "        # Divide total test loss by length of test dataloader (per batch)\n",
    "        test_loss /= len(test_dataloader)\n",
    "\n",
    "        # Divide total accuracy by length of test dataloader (per batch)\n",
    "        test_acc /= len(test_dataloader)\n",
    "\n",
    "    ## Print out what's happening\n",
    "    print(f\"\\nTrain loss: {train_loss:.5f} | Test loss: {test_loss:.5f}, Test acc: {test_acc:.2f}%\\n\")\n",
    "\n",
    "# Calculate training time      \n",
    "train_time_end_on_cpu = timer()\n",
    "total_train_time_model_0 = print_train_time(start=train_time_start_on_cpu, \n",
    "                                           end=train_time_end_on_cpu,\n",
    "                                           device=str(next(model_0.parameters()).device))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Our baseline model seems to have done ok as a start."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4. Making predictions and evaluating"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Let's make a function that encompasses the whole evaluating/testing process."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'model_name': 'FashionMNISTV0',\n",
       " 'model_loss': 0.47663894295692444,\n",
       " 'model_acc': 83.42651757188499}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.manual_seed(42)\n",
    "\n",
    "def eval_model(model:torch.nn.Module,\n",
    "                data_loader: torch.utils.data.DataLoader,\n",
    "                loss_fn: torch.nn.Module,\n",
    "                accuracy_fn):\n",
    "    loss, acc = 0, 0\n",
    "    model.eval()\n",
    "    with torch.inference_mode():\n",
    "        for X,y in data_loader:\n",
    "            y_pred = model(X)\n",
    "\n",
    "            loss += loss_fn(y_pred,y)\n",
    "            acc += accuracy_fn(y_true=y, y_pred=y_pred.argmax(dim=1))\n",
    "\n",
    "        loss /= len(data_loader)\n",
    "        acc /= len(data_loader)\n",
    "\n",
    "    return {\"model_name\": model.__class__.__name__, \n",
    "            \"model_loss\": loss.item(),\n",
    "            \"model_acc\": acc}\n",
    "\n",
    "model_0_results = eval_model(model=model_0, data_loader=test_dataloader,\n",
    "    loss_fn=loss_fn, accuracy_fn=accuracy_fn\n",
    ")\n",
    "model_0_results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### An easy method for a one step evaluation!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 5. Setup device"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### With Deep Learning, they are usually done with the help of a GPU.\n",
    "##### But as I mentioned previously, since I'm using a macbook, I will be using the MPS from Apple."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='mps')"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = torch.device(\"mps\") if torch.backends.mps.is_available() else torch.device('cpu')\n",
    "device"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 6. Adding non-linearity"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### We saw the power of adding non-linearity into our model in the previous notebook.\n",
    "##### So let us add the Relu function, nn.ReLU(), in between each linear layer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FashionMNISTModelV1(nn.Module):\n",
    "    def __init__(self,input_shape, hidden_units, output_shape):\n",
    "        super().__init__()\n",
    "        self.layer_stack = nn.Sequential(\n",
    "            nn.Flatten(),\n",
    "            nn.Linear(input_shape, output_shape),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(hidden_units, output_shape),\n",
    "            nn.ReLU()\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        return self.layer_stack(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Let's load our new model onto MPS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='mps', index=0)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.manual_seed(42)\n",
    "model_1 = FashionMNISTModelV1(784, 10, len(class_names)).to(device)\n",
    "next(model_1.parameters()).device"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Let's set up the loss, optimizer and evaluation metrics."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "from helper_functions import accuracy_fn\n",
    "loss_fn = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.SGD(params = model_1.parameters(),\n",
    "lr = 0.01)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training and Testing loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_step(model, data_loader, loss_fn, optimizer, accuracy_fn, device = device):\n",
    "    train_loss, train_acc = 0, 0\n",
    "    for batch, (X,y) in enumerate(data_loader):\n",
    "        X, y = X.to(device), y.to(device)\n",
    "        y_pred = model(X)\n",
    "        loss = loss_fn(y_pred, y)\n",
    "        train_loss+=loss\n",
    "        train_acc += accuracy_fn(y, y_pred.argmax(dim=1))\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        loss.backward()\n",
    "\n",
    "        optimizer.step()\n",
    "    \n",
    "    train_loss /= len(data_loader)\n",
    "    train_acc /= len(data_loader)\n",
    "    print(f\"Train Loss: {train_loss:.5f} | Train accuracy: {train_acc:.2f}%\")\n",
    "\n",
    "def test_step(data_loader, model, loss_fn, accuracy_fn, device=device):\n",
    "    test_loss, test_acc = 0, 0\n",
    "    model.eval()\n",
    "\n",
    "    with torch.inference_mode():\n",
    "        for X,y in data_loader:\n",
    "            X, y = X.to(device), y.to(device)\n",
    "\n",
    "            test_pred = model(X)\n",
    "\n",
    "            test_loss += loss_fn(test_pred, y)\n",
    "            test_acc += accuracy_fn(y, test_pred.argmax(dim=1))\n",
    "    \n",
    "        test_loss /= len(data_loader)\n",
    "        test_acc /= len(data_loader)\n",
    "        print(f\"Test loss: {test_loss:.5f} | Test accuracy: {test_acc:.2f}%\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Let's run the code using MPS and check how long it takes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/3 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0\n",
      "-----\n",
      "Train Loss: 0.90456 | Train accuracy: 67.08%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 33%|███▎      | 1/3 [00:05<00:11,  5.81s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test loss: 0.92040 | Test accuracy: 66.51%\n",
      "\n",
      "Epoch: 1\n",
      "-----\n",
      "Train Loss: 0.89381 | Train accuracy: 67.36%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 67%|██████▋   | 2/3 [00:11<00:05,  5.75s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test loss: 0.91260 | Test accuracy: 66.36%\n",
      "\n",
      "Epoch: 2\n",
      "-----\n",
      "Train Loss: 0.88470 | Train accuracy: 67.56%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 3/3 [00:17<00:00,  5.74s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test loss: 0.91010 | Test accuracy: 66.52%\n",
      "\n",
      "Train time on mps: 17.221 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "torch.manual_seed(42)\n",
    "\n",
    "from timeit import default_timer as timer\n",
    "train_time_start_on_gpu = timer()\n",
    "\n",
    "epochs = 3\n",
    "\n",
    "for epoch in tqdm(range(epochs)):\n",
    "    print(f\"Epoch: {epoch}\\n-----\")\n",
    "    train_step(data_loader = train_dataloader, model =model_1, \n",
    "    loss_fn = loss_fn, optimizer = optimizer, accuracy_fn = accuracy_fn)\n",
    "    test_step(data_loader=test_dataloader,\n",
    "        model=model_1,\n",
    "        loss_fn=loss_fn,\n",
    "        accuracy_fn=accuracy_fn\n",
    "    )\n",
    "\n",
    "train_time_end_on_gpu = timer()\n",
    "total_train_time_model_1 = print_train_time(start=train_time_start_on_gpu,\n",
    "                                            end=train_time_end_on_gpu,\n",
    "                                            device=device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Now, let's evaluate our model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'model_name': 'FashionMNISTModelV1',\n",
       " 'model_loss': 0.910102903842926,\n",
       " 'model_acc': 66.52356230031948}"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.manual_seed(42)\n",
    "def eval_model(model: torch.nn.Module, \n",
    "               data_loader: torch.utils.data.DataLoader, \n",
    "               loss_fn: torch.nn.Module, \n",
    "               accuracy_fn, \n",
    "               device: torch.device = device):\n",
    "    \"\"\"Evaluates a given model on a given dataset.\n",
    "\n",
    "    Args:\n",
    "        model (torch.nn.Module): A PyTorch model capable of making predictions on data_loader.\n",
    "        data_loader (torch.utils.data.DataLoader): The target dataset to predict on.\n",
    "        loss_fn (torch.nn.Module): The loss function of model.\n",
    "        accuracy_fn: An accuracy function to compare the models predictions to the truth labels.\n",
    "        device (str, optional): Target device to compute on. Defaults to device.\n",
    "\n",
    "    Returns:\n",
    "        (dict): Results of model making predictions on data_loader.\n",
    "    \"\"\"\n",
    "    loss, acc = 0, 0\n",
    "    model.eval()\n",
    "    with torch.inference_mode():\n",
    "        for X, y in data_loader:\n",
    "            # Send data to the target device\n",
    "            X, y = X.to(device), y.to(device)\n",
    "            y_pred = model(X)\n",
    "            loss += loss_fn(y_pred, y)\n",
    "            acc += accuracy_fn(y_true=y, y_pred=y_pred.argmax(dim=1))\n",
    "        \n",
    "        # Scale loss and acc\n",
    "        loss /= len(data_loader)\n",
    "        acc /= len(data_loader)\n",
    "    return {\"model_name\": model.__class__.__name__, # only works when model was created with a class\n",
    "            \"model_loss\": loss.item(),\n",
    "            \"model_acc\": acc}\n",
    "\n",
    "# Calculate model 1 results with device-agnostic code \n",
    "model_1_results = eval_model(model=model_1, data_loader=test_dataloader,\n",
    "    loss_fn=loss_fn, accuracy_fn=accuracy_fn,\n",
    "    device=device\n",
    ")\n",
    "model_1_results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### It looks like our model is not performing better than the original, basic model.\n",
    "##### It seems like it's about time we learn about the Convolutional Neural Network (CNN)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### The typical structure would look something like this:\n",
    "##### Input layer -> Convolutional Layer -> Activation layer -> pooling layer -> Output layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Cnn(\n",
       "  (block_1): Sequential(\n",
       "    (0): Conv2d(1, 10, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (1): ReLU()\n",
       "    (2): Conv2d(10, 10, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (3): ReLU()\n",
       "    (4): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "  )\n",
       "  (block_2): Sequential(\n",
       "    (0): Conv2d(10, 10, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (1): ReLU()\n",
       "    (2): Conv2d(10, 10, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (3): ReLU()\n",
       "    (4): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "  )\n",
       "  (classifier): Sequential(\n",
       "    (0): Flatten(start_dim=1, end_dim=-1)\n",
       "    (1): Linear(in_features=490, out_features=10, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class Cnn(nn.Module):\n",
    "    def __init__(self, input_shape, hidden_units, output_shape):\n",
    "        super().__init__()\n",
    "        self.block_1 = nn.Sequential(\n",
    "            nn.Conv2d(input_shape,hidden_units,3,1,1), # 28 x 28 x hidden_units\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(hidden_units, hidden_units, 3,1,1), # 28 x 28 x hidden_units\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool2d(kernel_size=2, stride=2) # 14 x 14 x hidden_units\n",
    "        )\n",
    "        self.block_2 = nn.Sequential(\n",
    "            nn.Conv2d(hidden_units, hidden_units, 3, padding=1), # 14 x 14 x hidden_units\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(hidden_units, hidden_units, 3, padding=1), # 14 x 14 x hidden_units\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool2d(2) # 7 x 7 x hidden_units\n",
    "        )\n",
    "        self.classifier = nn.Sequential(\n",
    "            nn.Flatten(),\n",
    "            # Where did this in_features shape come from? \n",
    "            # It's because each layer of our network compresses and changes the shape of our inputs data.\n",
    "            nn.Linear(in_features=hidden_units*7*7, \n",
    "                      out_features=output_shape)\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.block_1(x)\n",
    "        x = self.block_2(x)\n",
    "        x = self.classifier(x)\n",
    "        return x\n",
    "\n",
    "torch.manual_seed(42)\n",
    "model_2 = Cnn(input_shape = 1, hidden_units = 10, output_shape = len(class_names)).to(device)\n",
    "model_2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### It is really important to keep track of the total number of features that is being calculated throughout the model.\n",
    "##### You will need to write down how many nodes there are when you flatten them in the end."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_fn = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.SGD(params = model_2.parameters(), lr = 0.1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Let's put our CNN model to the test!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/3 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Loss: 0.57906 | Train accuracy: 79.07%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 33%|███▎      | 1/3 [00:10<00:21, 11.00s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test loss: 0.38724 | Test accuracy: 85.61%\n",
      "\n",
      "Train Loss: 0.35557 | Train accuracy: 87.28%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 67%|██████▋   | 2/3 [00:21<00:10, 10.48s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test loss: 0.36830 | Test accuracy: 86.25%\n",
      "\n",
      "Train Loss: 0.31745 | Train accuracy: 88.50%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 3/3 [00:31<00:00, 10.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test loss: 0.32704 | Test accuracy: 88.13%\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "epochs = 3\n",
    "for epoch in tqdm(range(epochs)):\n",
    "    train_step(data_loader = train_dataloader,\n",
    "                model = model_2,\n",
    "                loss_fn = loss_fn,\n",
    "                optimizer=optimizer,\n",
    "                accuracy_fn=accuracy_fn,\n",
    "                device = device)\n",
    "    test_step(data_loader=test_dataloader,\n",
    "                    model = model_2,\n",
    "                    loss_fn=loss_fn,\n",
    "                    accuracy_fn=accuracy_fn,\n",
    "                    device = device)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### It seems like our model improved by a lot!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'model_name': 'Cnn',\n",
       " 'model_loss': 0.3270363211631775,\n",
       " 'model_acc': 88.12899361022365}"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get model_2 results \n",
    "model_2_results = eval_model(\n",
    "    model=model_2,\n",
    "    data_loader=test_dataloader,\n",
    "    loss_fn=loss_fn,\n",
    "    accuracy_fn=accuracy_fn\n",
    ")\n",
    "model_2_results"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10.8 ('pytorch': venv)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "40ceb6fe12160d1b46a968bef1533ff09c8e7e4d23d39b120682f3311adef91d"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}