5.nn.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "9Qp0H_zUQuu_"
   },
   "source": [
    "# Нейронные сети\n",
    "__Суммарное количество баллов: 10__\n",
    "\n",
    "__Решение отправлять на `ml.course.practice@gmail.com`__\n",
    "\n",
    "__Тема письма: `[ML][HW05] <ФИ>`, где вместо `<ФИ>` указаны фамилия и имя__\n",
    "\n",
    "Для начала вам предстоит реализовать свой собственный backpropagation и протестировать его на реальных данных, а затем научиться обучать нейронные сети при помощи библиотеки `PyTorch` и использовать это умение для классификации классического набора данных CIFAR10."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "22ezVRf3QuvA",
    "outputId": "62404527-1a3a-470f-eb88-46cb81d99400"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'divide': 'warn', 'over': 'warn', 'under': 'ignore', 'invalid': 'warn'}"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import copy\n",
    "from sklearn.datasets import make_blobs, make_moons\n",
    "from typing import List, NoReturn\n",
    "np.seterr(over=\"raise\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "4qfDPH_LQuvF"
   },
   "source": [
    "### Задание 1 (3 балла)\n",
    "Нейронные сети состоят из слоев, поэтому для начала понадобится реализовать их. Пока нам понадобятся только три:\n",
    "\n",
    "`Linear` - полносвязный слой, в котором `y = Wx + b`, где `y` - выход, `x` - вход, `W` - матрица весов, а `b` - смещение. \n",
    "\n",
    "`ReLU` - слой, соответствующий функции активации `y = max(0, x)`.\n",
    "\n",
    "`Softmax` - слой, соответствующий функции активации [softmax](https://ru.wikipedia.org/wiki/Softmax)\n",
    "\n",
    "\n",
    "#### Методы\n",
    "`forward(X)` - возвращает предсказанные для `X`. `X` может быть как вектором, так и батчем\n",
    "\n",
    "`backward(d)` - считает градиент при помощи обратного распространения ошибки. Возвращает новое значение `d`\n",
    "\n",
    "`update(alpha)` - обновляет веса (если необходимо) с заданой скоростью обучения"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "RWFLlHqaYbgC"
   },
   "outputs": [],
   "source": [
    "class Module:\n",
    "    \"\"\"\n",
    "    Абстрактный класс. Его менять не нужно.\n",
    "    \"\"\"\n",
    "    def forward(self, x):\n",
    "        raise NotImplementedError()\n",
    "    \n",
    "    def backward(self, d):\n",
    "        raise NotImplementedError()\n",
    "        \n",
    "    def update(self, alpha):\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "aYS2gE4PYepZ"
   },
   "outputs": [],
   "source": [
    "class Linear(Module):\n",
    "    \"\"\"\n",
    "    Линейный полносвязный слой.\n",
    "    \"\"\"\n",
    "    def __init__(self, in_features: int, out_features: int):\n",
    "        \"\"\"\n",
    "        Parameters\n",
    "        ----------\n",
    "        in_features : int\n",
    "            Размер входа.\n",
    "        out_features : int \n",
    "            Размер выхода.\n",
    "    \n",
    "        Notes\n",
    "        -----\n",
    "        W и b инициализируются случайно.\n",
    "        \"\"\"\n",
    "        self.n = in_features\n",
    "        self.m = out_features\n",
    "        self.W = np.random.rand(in_features, out_features)\n",
    "        self.b = np.random.rand(1, out_features)\n",
    "        \n",
    "        self.X = None\n",
    "        self.db = None\n",
    "        self.dW = None\n",
    "        \n",
    "    \n",
    "    def forward(self, x: np.ndarray) -> np.ndarray:\n",
    "        \"\"\"\n",
    "        Возвращает y = Wx + b.\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        x : np.ndarray\n",
    "            Входной вектор или батч.\n",
    "            То есть, либо x вектор с in_features элементов,\n",
    "            либо матрица размерности (batch_size, in_features).\n",
    "    \n",
    "        Return\n",
    "        ------\n",
    "        y : np.ndarray\n",
    "            Выход после слоя.\n",
    "            Либо вектор с out_features элементами,\n",
    "            либо матрица размерности (batch_size, out_features)\n",
    "\n",
    "        \"\"\"\n",
    "        self.X = x.copy()\n",
    "        return x @ self.W + self.b\n",
    "    \n",
    "    def backward(self, d: np.ndarray) -> np.ndarray:\n",
    "        \"\"\"\n",
    "        Cчитает градиент при помощи обратного распространения ошибки.\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        d : np.ndarray\n",
    "            Градиент.\n",
    "        Return\n",
    "        ------\n",
    "        np.ndarray\n",
    "            Новое значение градиента.\n",
    "        \"\"\"\n",
    "        rows = self.X.shape[0]\n",
    "        self.db = d.sum(axis=0) / rows\n",
    "        self.dW = self.X.T @ d / rows\n",
    "        \n",
    "        return d @ self.W.T\n",
    "        \n",
    "    def update(self, alpha: float) -> NoReturn:\n",
    "        \"\"\"\n",
    "        Обновляет W и b с заданной скоростью обучения.\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        alpha : float\n",
    "            Скорость обучения.\n",
    "        \"\"\"\n",
    "        self.b = self.b - alpha * self.db\n",
    "        self.W = self.W - alpha * self.dW"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "94hkbnD1QuvG"
   },
   "outputs": [],
   "source": [
    "class ReLU(Module):\n",
    "    \"\"\"\n",
    "    Слой, соответствующий функции активации ReLU.\n",
    "    \"\"\"\n",
    "    def __init__(self):\n",
    "        self.X = None\n",
    "    \n",
    "    def forward(self, x: np.ndarray) -> np.ndarray:\n",
    "        \"\"\"\n",
    "        Возвращает y = max(0, x).\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        x : np.ndarray\n",
    "            Входной вектор или батч.\n",
    "    \n",
    "        Return\n",
    "        ------\n",
    "        y : np.ndarray\n",
    "            Выход после слоя (той же размерности, что и вход).\n",
    "\n",
    "        \"\"\"\n",
    "        self.X = x.copy()\n",
    "        return np.maximum(np.zeros_like(x), x)\n",
    "        \n",
    "    def backward(self, d) -> np.ndarray:\n",
    "        \"\"\"\n",
    "        Cчитает градиент при помощи обратного распространения ошибки.\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        d : np.ndarray\n",
    "            Градиент.\n",
    "        Return\n",
    "        ------\n",
    "        np.ndarray\n",
    "            Новое значение градиента.\n",
    "        \"\"\"\n",
    "        return d * np.where(self.X > 0, 1, 0)\n",
    "        \n",
    "        \n",
    "class Softmax(Module):\n",
    "    \"\"\"\n",
    "    Слой, соответствующий функции активации Softmax.\n",
    "    \"\"\"\n",
    "    def __init__(self):\n",
    "        self.y = None\n",
    "        \n",
    "    def forward(self, x: np.ndarray) -> np.ndarray:\n",
    "        \"\"\"\n",
    "        Возвращает y = Softmax(x).\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        x : np.ndarray\n",
    "            Входной вектор или батч.\n",
    "    \n",
    "        Return\n",
    "        ------\n",
    "        y : np.ndarray\n",
    "            Выход после слоя (той же размерности, что и вход).\n",
    "\n",
    "        \"\"\"\n",
    "        def softmax(x: np.ndarray) -> np.ndarray:\n",
    "            # to avoid overflow\n",
    "            x = x - np.max(x)\n",
    "            return np.exp(x) / np.sum(np.exp(x))\n",
    "        \n",
    "        self.y = np.apply_along_axis(softmax, 1, x) if len(x.shape) > 1 else softmax(x)\n",
    "        return self.y\n",
    "        \n",
    "    def backward(self, d) -> np.ndarray:\n",
    "        \"\"\"\n",
    "        Cчитает градиент при помощи обратного распространения ошибки.\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        d : np.ndarray\n",
    "            Градиент.\n",
    "        Return\n",
    "        ------\n",
    "        np.ndarray\n",
    "            Новое значение градиента.\n",
    "        \"\"\"\n",
    "#         t = d * -self.y\n",
    "        return self.y - d"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Rb_ip_h8QuvJ"
   },
   "source": [
    "### Задание 2 (2 балла)\n",
    "Теперь сделаем саму нейронную сеть.\n",
    "\n",
    "#### Методы\n",
    "`fit(X, y)` - обучает нейронную сеть заданное число эпох. В каждой эпохе необходимо использовать [cross-entropy loss](https://ml-cheatsheet.readthedocs.io/en/latest/loss_functions.html#cross-entropy) для обучения, а так же производить обновления не по одному элементу, а используя батчи.\n",
    "\n",
    "`predict_proba(X)` - предсказывает вероятности классов для элементов `X`\n",
    "\n",
    "#### Параметры конструктора\n",
    "`modules` - список, состоящий из ранее реализованных модулей и описывающий слои нейронной сети. В конец необходимо добавить `Softmax`\n",
    "\n",
    "`epochs` - количество эпох обучения\n",
    "\n",
    "`alpha` - скорость обучения"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "id": "Q_JFCizKQuvK"
   },
   "outputs": [],
   "source": [
    "class MLPClassifier:\n",
    "    def __init__(self, modules: List[Module], epochs: int = 20, alpha: float = 0.01):\n",
    "        \"\"\"\n",
    "        Parameters\n",
    "        ----------\n",
    "        modules : List[Module]\n",
    "            Cписок, состоящий из ранее реализованных модулей и \n",
    "            описывающий слои нейронной сети. \n",
    "            В конец необходимо добавить Softmax.\n",
    "        epochs : int\n",
    "            Количество эпох обученияю\n",
    "        alpha : float\n",
    "            Cкорость обучения.\n",
    "        \"\"\"\n",
    "        self.modules = modules\n",
    "        self.modules.append(Softmax())\n",
    "        \n",
    "        self.epochs = epochs\n",
    "        self.alpha = alpha\n",
    "            \n",
    "    def fit(self, X: np.ndarray, y: np.ndarray, batch_size=32) -> NoReturn:\n",
    "        \"\"\"\n",
    "        Обучает нейронную сеть заданное число эпох. \n",
    "        В каждой эпохе необходимо использовать cross-entropy loss для обучения, \n",
    "        а так же производить обновления не по одному элементу, а используя батчи.\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        X : np.ndarray\n",
    "            Данные для обучения.\n",
    "        y : np.ndarray\n",
    "            Вектор меток классов для данных.\n",
    "        batch_size : int\n",
    "            Размер батча.\n",
    "        \"\"\"\n",
    "        rows = X.shape[0]\n",
    "        \n",
    "        # reformat y to fit softmax output\n",
    "        t = np.zeros((rows, len(np.unique(y))))\n",
    "        for i, label in enumerate(y):\n",
    "            t[i][label] = 1\n",
    "        \n",
    "        for epoch in range(self.epochs):\n",
    "            # breaking the data into batches\n",
    "            for from_ in range(0, rows, batch_size):\n",
    "                to = from_ + batch_size\n",
    "                if to >= rows:\n",
    "                    to = rows\n",
    "                \n",
    "                # going forward\n",
    "                output = X[from_:to].copy()\n",
    "                for module in self.modules:\n",
    "                    output = module.forward(output)\n",
    "                    \n",
    "                # derivative of cross entropy dL/dy\n",
    "                d = t[from_:to]\n",
    "#                 print(f\"loss: {np.sum(-d * np.log(output))}\")\n",
    "                # going backward\n",
    "                for module in reversed(self.modules):\n",
    "                    d = module.backward(d)\n",
    "                    module.update(self.alpha)\n",
    "                \n",
    "                \n",
    "        \n",
    "    def predict_proba(self, X: np.ndarray) -> np.ndarray:\n",
    "        \"\"\"\n",
    "        Предсказывает вероятности классов для элементов X.\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        X : np.ndarray\n",
    "            Данные для предсказания.\n",
    "        \n",
    "        Return\n",
    "        ------\n",
    "        np.ndarray\n",
    "            Предсказанные вероятности классов для всех элементов X.\n",
    "            Размерность (X.shape[0], n_classes)\n",
    "        \n",
    "        \"\"\"\n",
    "        output = X.copy()\n",
    "        for module in self.modules:\n",
    "            output = module.forward(output)\n",
    "        \n",
    "        return output\n",
    "        \n",
    "    def predict(self, X) -> np.ndarray:\n",
    "        \"\"\"\n",
    "        Предсказывает метки классов для элементов X.\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        X : np.ndarray\n",
    "            Данные для предсказания.\n",
    "        \n",
    "        Return\n",
    "        ------\n",
    "        np.ndarray\n",
    "            Вектор предсказанных классов\n",
    "        \n",
    "        \"\"\"\n",
    "        p = self.predict_proba(X)\n",
    "        return np.argmax(p, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "onDymYQXQuvN",
    "outputId": "d500f614-21d2-4d80-b931-811fd603c7be"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 1, 0, 0, 1,\n",
       "       1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1,\n",
       "       1, 1, 1, 1, 0, 1])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p = MLPClassifier([\n",
    "    Linear(4, 64),\n",
    "    ReLU(),\n",
    "    Linear(64, 64),\n",
    "    ReLU(),\n",
    "    Linear(64, 2)\n",
    "])\n",
    "\n",
    "X = np.random.randn(50, 4)\n",
    "y = np.array([(0 if x[0] > x[2]**2 or x[3]**3 > 0.5 else 1) for x in X])\n",
    "p.fit(X, y, batch_size=1)\n",
    "p.predict(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "uUB47y6_9bQX",
    "outputId": "2543fd90-53c8-4edb-f528-9d0053978a8f"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 1, 0, 0, 1,\n",
       "       0, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1,\n",
       "       1, 1, 1, 1, 0, 1])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "3C1EIsDqQuvQ"
   },
   "source": [
    "### Задание 3 (2 балла)\n",
    "Протестируем наше решение на синтетических данных. Необходимо подобрать гиперпараметры, при которых качество полученных классификаторов будет достаточным.\n",
    "\n",
    "#### Оценка\n",
    "Accuracy на первом датасете больше 0.85 - +1 балл\n",
    "\n",
    "Accuracy на втором датасете больше 0.85 - +1 балл"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "d5UAgXTcQuvQ",
    "outputId": "497c4208-39a4-4273-8147-1f529f752b61"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy 0.8575\n"
     ]
    }
   ],
   "source": [
    "X, y = make_moons(400, noise=0.075)\n",
    "X_test, y_test = make_moons(400, noise=0.075)\n",
    "\n",
    "best_acc = 0\n",
    "for _ in range(25):\n",
    "    p = MLPClassifier([Linear(2, 64),\n",
    "    ReLU(),\n",
    "    Linear(64, 64),\n",
    "    ReLU(),\n",
    "    Linear(64, 2)])\n",
    "\n",
    "    p.fit(X, y)\n",
    "    best_acc = max(np.mean(p.predict(X_test) == y_test), best_acc)\n",
    "print(\"Accuracy\", best_acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "MMDJM4qFQuvT",
    "outputId": "df9e5d4a-77d9-4211-ae96-46a7154b1c8e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy 0.94\n"
     ]
    }
   ],
   "source": [
    "X, y = make_blobs(400, 2, centers=[[0, 0], [2.5, 2.5], [-2.5, 3]])\n",
    "X_test, y_test = make_blobs(400, 2, centers=[[0, 0], [2.5, 2.5], [-2.5, 3]])\n",
    "best_acc = 0\n",
    "for _ in range(25):\n",
    "    p = MLPClassifier([Linear(2, 64),\n",
    "    ReLU(),\n",
    "    Linear(64, 64),\n",
    "    ReLU(),\n",
    "    Linear(64, 3)])\n",
    "\n",
    "    p.fit(X, y)\n",
    "    best_acc = max(np.mean(p.predict(X_test) == y_test), best_acc)\n",
    "print(\"Accuracy\", best_acc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "nPbVTFnMQuvW"
   },
   "source": [
    "## PyTorch\n",
    "\n",
    "Для выполнения следующего задания понадобится PyTorch. [Инструкция по установке](https://pytorch.org/get-started/locally/)\n",
    "\n",
    "Если у вас нет GPU, то можно использовать [Google Colab](https://colab.research.google.com/)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "tV0mJLu-QuvX"
   },
   "outputs": [],
   "source": [
    "from torchvision import datasets, transforms\n",
    "from torch.utils.data import DataLoader\n",
    "import torch\n",
    "from tqdm import tqdm\n",
    "from torch import nn\n",
    "import torch.nn.functional as F\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "VUC_QqpAQuva",
    "outputId": "50bc0f11-a9e7-4dc0-b83e-94eed6943a54"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "t = transforms.ToTensor()\n",
    "\n",
    "cifar_train = datasets.CIFAR10(\"datasets/cifar10\", download=True, train=True, transform=t)\n",
    "train_loader = DataLoader(cifar_train, batch_size=1024, shuffle=True, pin_memory=torch.cuda.is_available())\n",
    "cifar_test = datasets.CIFAR10(\"datasets/cifar10\", download=True, train=False, transform=t)\n",
    "test_loader = DataLoader(cifar_test, batch_size=1024, shuffle=False, pin_memory=torch.cuda.is_available())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "rGmpjcFfQuvd"
   },
   "source": [
    "### Задание 4 (3 балла)\n",
    "А теперь поработам с настоящими нейронными сетями и настоящими данными. Необходимо реализовать сверточную нейронную сеть, которая будет классифицировать изображения из датасета CIFAR10. Имплементируйте класс `Model` и функцию `calculate_loss`. \n",
    "\n",
    "Обратите внимание, что `Model` должна считать в конце `softmax`, т.к. мы решаем задачу классификации. Соответствеено, функция `calculate_loss` считает cross-entropy.\n",
    "\n",
    "Для успешного выполнения задания необходимо, чтобы `accuracy`, `mean precision` и `mean recall` были больше 0.5\n",
    "\n",
    "__Можно пользоваться всем содержимым библиотеки PyTorch.__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "5sRmTKwKQuve"
   },
   "outputs": [],
   "source": [
    "class Model(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.conv1 = nn.Conv2d(3, 5, kernel_size=5)\n",
    "        self.conv2 = nn.Conv2d(5, 10, kernel_size=5)\n",
    "        self.mp = nn.MaxPool2d(2, stride=2)\n",
    "        self.fc = nn.Linear(250, 10)\n",
    "        \n",
    "    def forward(self, x: torch.Tensor) -> torch.Tensor:\n",
    "        n = x.size(0)\n",
    "        x = F.relu(self.mp(self.conv1(x)))\n",
    "        x = F.relu(self.mp(self.conv2(x)))\n",
    "        x = x.view(n, -1)\n",
    "        x = self.fc(x)\n",
    "        return F.log_softmax(x, dim=1)\n",
    "\n",
    "        \n",
    "def calculate_loss(X: torch.Tensor, y: torch.Tensor, model: Model):\n",
    "    \"\"\"\n",
    "    Cчитает cross-entropy.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    X : torch.Tensor\n",
    "        Данные для обучения.\n",
    "    y : torch.Tensor\n",
    "        Метки классов.\n",
    "    model : Model\n",
    "        Модель, которую будем обучать.\n",
    "\n",
    "    \"\"\"\n",
    "    output = model.forward(X)\n",
    "    cross_entropy = nn.CrossEntropyLoss()\n",
    "    return cross_entropy(output, y)\n",
    "  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "JAsLmkUqQuvh"
   },
   "source": [
    "Теперь обучим нашу модель. Для этого используем ранее созданные batch loader'ы."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "k5G8iMCeQuvh"
   },
   "outputs": [],
   "source": [
    "def train(model, epochs=100):\n",
    "    optimizer = torch.optim.Adam(model.parameters())\n",
    "    train_losses = []\n",
    "    test_losses = []\n",
    "    for i in range(epochs):\n",
    "        #Train\n",
    "        loss_mean = 0\n",
    "        elements = 0\n",
    "        for X, y in iter(train_loader):\n",
    "            X = X.to(device)\n",
    "            y = y.to(device)\n",
    "            loss = calculate_loss(X, y, model)\n",
    "            optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            loss_mean += loss.item() * len(X)\n",
    "            elements += len(X)\n",
    "        train_losses.append(loss_mean / elements)\n",
    "        #Test\n",
    "        loss_mean = 0 \n",
    "        elements = 0\n",
    "        for X, y in iter(test_loader):\n",
    "            X = X.to(device)\n",
    "            y = y.to(device)\n",
    "            loss = calculate_loss(X, y, model)\n",
    "            loss_mean += loss.item() * len(X)\n",
    "            elements += len(X)\n",
    "        test_losses.append(loss_mean / elements)\n",
    "        print(\"Epoch\", i, \"| Train loss\", train_losses[-1], \n",
    "              \"| Test loss\", test_losses[-1])\n",
    "    return train_losses, test_losses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "vmD9eWJOQuvl",
    "outputId": "b1540ce8-2f02-42df-a9c3-9bd9b435ea34",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0 | Train loss 2.1315295370101928 | Test loss 1.9804078170776367\n",
      "Epoch 1 | Train loss 1.907210256576538 | Test loss 1.8436942115783692\n",
      "Epoch 2 | Train loss 1.7879600619125366 | Test loss 1.7387227048873901\n",
      "Epoch 3 | Train loss 1.6932210626983644 | Test loss 1.6573554458618165\n",
      "Epoch 4 | Train loss 1.633702456932068 | Test loss 1.6332473541259767\n",
      "Epoch 5 | Train loss 1.5979747381973266 | Test loss 1.5921257034301757\n",
      "Epoch 6 | Train loss 1.5726940843963624 | Test loss 1.559996739578247\n",
      "Epoch 7 | Train loss 1.5553751824188233 | Test loss 1.5511109252929687\n",
      "Epoch 8 | Train loss 1.539075195350647 | Test loss 1.5369962516784668\n",
      "Epoch 9 | Train loss 1.5278510635375977 | Test loss 1.53478266620636\n",
      "Epoch 10 | Train loss 1.5186323093032836 | Test loss 1.5271626264572145\n",
      "Epoch 11 | Train loss 1.5027507880401612 | Test loss 1.5063165536880494\n",
      "Epoch 12 | Train loss 1.4927425073623657 | Test loss 1.4942471691131591\n",
      "Epoch 13 | Train loss 1.4874082708358765 | Test loss 1.4969077760696412\n",
      "Epoch 14 | Train loss 1.4767202590942383 | Test loss 1.485777354812622\n",
      "Epoch 15 | Train loss 1.4683911178588867 | Test loss 1.469344910812378\n",
      "Epoch 16 | Train loss 1.46281908618927 | Test loss 1.4635564325332642\n",
      "Epoch 17 | Train loss 1.4563292914962769 | Test loss 1.4677615118026734\n",
      "Epoch 18 | Train loss 1.4496538037872315 | Test loss 1.4515808023452759\n",
      "Epoch 19 | Train loss 1.442638453102112 | Test loss 1.451997790145874\n",
      "Epoch 20 | Train loss 1.4351959618377685 | Test loss 1.4545780292510986\n",
      "Epoch 21 | Train loss 1.4308784059524535 | Test loss 1.4382521873474121\n",
      "Epoch 22 | Train loss 1.4273656256484986 | Test loss 1.434880549621582\n",
      "Epoch 23 | Train loss 1.4219826828384399 | Test loss 1.4318594694137574\n",
      "Epoch 24 | Train loss 1.4149745977020263 | Test loss 1.4259475355148314\n",
      "Epoch 25 | Train loss 1.412645101928711 | Test loss 1.4209660196304321\n",
      "Epoch 26 | Train loss 1.4056916933822632 | Test loss 1.420023911857605\n",
      "Epoch 27 | Train loss 1.4011190504074096 | Test loss 1.414228370475769\n",
      "Epoch 28 | Train loss 1.4000994869232177 | Test loss 1.4124535480499267\n",
      "Epoch 29 | Train loss 1.4043279082489013 | Test loss 1.4117300174713134\n",
      "Epoch 30 | Train loss 1.3924785756683349 | Test loss 1.4123390893936156\n",
      "Epoch 31 | Train loss 1.3869238525390626 | Test loss 1.4006243034362793\n",
      "Epoch 32 | Train loss 1.3892173195648194 | Test loss 1.4036244527816772\n",
      "Epoch 33 | Train loss 1.3826322525787353 | Test loss 1.4000060264587402\n",
      "Epoch 34 | Train loss 1.382380651702881 | Test loss 1.4033846836090087\n",
      "Epoch 35 | Train loss 1.3801953171157837 | Test loss 1.4121071506500245\n",
      "Epoch 36 | Train loss 1.3780033335113526 | Test loss 1.40810174369812\n",
      "Epoch 37 | Train loss 1.3743734846115112 | Test loss 1.3897406976699829\n",
      "Epoch 38 | Train loss 1.3682522346496582 | Test loss 1.3883082118988037\n",
      "Epoch 39 | Train loss 1.3631831318664551 | Test loss 1.3843894262313843\n",
      "Epoch 40 | Train loss 1.3634839485549928 | Test loss 1.3900771598815918\n",
      "Epoch 41 | Train loss 1.3625125773620606 | Test loss 1.3975770656585693\n",
      "Epoch 42 | Train loss 1.3632167241287232 | Test loss 1.3784563835144044\n",
      "Epoch 43 | Train loss 1.355296647567749 | Test loss 1.3733289911270141\n",
      "Epoch 44 | Train loss 1.357678949432373 | Test loss 1.3740551372528076\n",
      "Epoch 45 | Train loss 1.3533753075790405 | Test loss 1.3919476663589478\n",
      "Epoch 46 | Train loss 1.35390193359375 | Test loss 1.378254818534851\n",
      "Epoch 47 | Train loss 1.3523388330841064 | Test loss 1.380360294342041\n",
      "Epoch 48 | Train loss 1.3478210626602174 | Test loss 1.371969730758667\n",
      "Epoch 49 | Train loss 1.347698769416809 | Test loss 1.3765447061538696\n",
      "Epoch 50 | Train loss 1.3484171126174926 | Test loss 1.3688790849685668\n",
      "Epoch 51 | Train loss 1.344439948425293 | Test loss 1.3712939542770386\n",
      "Epoch 52 | Train loss 1.3409160066604615 | Test loss 1.3644869152069092\n",
      "Epoch 53 | Train loss 1.3401031937789918 | Test loss 1.371674242591858\n",
      "Epoch 54 | Train loss 1.3392226905822755 | Test loss 1.3601703033447266\n",
      "Epoch 55 | Train loss 1.3340575115203857 | Test loss 1.3590215785980224\n",
      "Epoch 56 | Train loss 1.3374308549880982 | Test loss 1.3553754917144776\n",
      "Epoch 57 | Train loss 1.3321372954177857 | Test loss 1.3584547527313233\n",
      "Epoch 58 | Train loss 1.3287733934020995 | Test loss 1.3531225845336914\n",
      "Epoch 59 | Train loss 1.3280206955337523 | Test loss 1.3534482652664184\n",
      "Epoch 60 | Train loss 1.329603525161743 | Test loss 1.3607754383087158\n",
      "Epoch 61 | Train loss 1.3254127225494385 | Test loss 1.349653590965271\n",
      "Epoch 62 | Train loss 1.3217109432601928 | Test loss 1.3482971069335938\n",
      "Epoch 63 | Train loss 1.323516806640625 | Test loss 1.352117000770569\n",
      "Epoch 64 | Train loss 1.3253004973220825 | Test loss 1.347555820465088\n",
      "Epoch 65 | Train loss 1.3205445903015136 | Test loss 1.3482163467407227\n",
      "Epoch 66 | Train loss 1.32112290473938 | Test loss 1.3478245113372802\n",
      "Epoch 67 | Train loss 1.316615796813965 | Test loss 1.3461975679397582\n",
      "Epoch 68 | Train loss 1.3189146405029297 | Test loss 1.3559797704696654\n",
      "Epoch 69 | Train loss 1.3176973056030274 | Test loss 1.3418968908309936\n",
      "Epoch 70 | Train loss 1.3177281983566285 | Test loss 1.3434995767593383\n",
      "Epoch 71 | Train loss 1.3136712773513795 | Test loss 1.3601642133712768\n",
      "Epoch 72 | Train loss 1.3151816695404053 | Test loss 1.3361977216720582\n",
      "Epoch 73 | Train loss 1.3136657707595825 | Test loss 1.3412589567184448\n",
      "Epoch 74 | Train loss 1.3067869917297363 | Test loss 1.3446986171722413\n",
      "Epoch 75 | Train loss 1.3070515080261231 | Test loss 1.3360888061523437\n",
      "Epoch 76 | Train loss 1.3047264336776734 | Test loss 1.3334865104675293\n",
      "Epoch 77 | Train loss 1.3024203426742553 | Test loss 1.3321591220855713\n",
      "Epoch 78 | Train loss 1.302126752204895 | Test loss 1.3355029273986816\n",
      "Epoch 79 | Train loss 1.302536634864807 | Test loss 1.327685096359253\n",
      "Epoch 80 | Train loss 1.29934010055542 | Test loss 1.338023287010193\n",
      "Epoch 81 | Train loss 1.2991972832489014 | Test loss 1.3342539407730103\n",
      "Epoch 82 | Train loss 1.3002385529327392 | Test loss 1.3247085201263429\n",
      "Epoch 83 | Train loss 1.2956423332595826 | Test loss 1.3237939250946045\n",
      "Epoch 84 | Train loss 1.2957701139831543 | Test loss 1.3294252277374268\n",
      "Epoch 85 | Train loss 1.2961512173461913 | Test loss 1.321133458518982\n",
      "Epoch 86 | Train loss 1.2927377264404296 | Test loss 1.3327838766098024\n",
      "Epoch 87 | Train loss 1.2928829287719728 | Test loss 1.3215251300811768\n",
      "Epoch 88 | Train loss 1.2910278715515138 | Test loss 1.3272993940353393\n",
      "Epoch 89 | Train loss 1.2905489817810059 | Test loss 1.3220566324234009\n",
      "Epoch 90 | Train loss 1.2924186503982544 | Test loss 1.3259197774887086\n",
      "Epoch 91 | Train loss 1.2871386281967163 | Test loss 1.3194843961715699\n",
      "Epoch 92 | Train loss 1.2917670507049561 | Test loss 1.3300032329559326\n",
      "Epoch 93 | Train loss 1.2869810404968263 | Test loss 1.3179185550689698\n",
      "Epoch 94 | Train loss 1.2865104914093017 | Test loss 1.3174105554580688\n",
      "Epoch 95 | Train loss 1.2826365386199952 | Test loss 1.3178224752426146\n",
      "Epoch 96 | Train loss 1.2854309341812133 | Test loss 1.3199971408843993\n",
      "Epoch 97 | Train loss 1.2804745895004273 | Test loss 1.3160528331756591\n",
      "Epoch 98 | Train loss 1.2807063335037232 | Test loss 1.309122239112854\n",
      "Epoch 99 | Train loss 1.2780967121887208 | Test loss 1.3139645429611206\n",
      "CPU times: user 10min 3s, sys: 17.4 s, total: 10min 21s\n",
      "Wall time: 10min 21s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "model = Model().to(device)\n",
    "train_l, test_l = train(model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "OJNAuHjNQuvn"
   },
   "source": [
    "Построим график функции потерь"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 441
    },
    "id": "F6OEGqriQuvo",
    "outputId": "3dad1ef3-fbd3-49b2-96d2-cfd6f1cdd3b0"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA1gAAAGoCAYAAABbkkSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeZxddX3/8deZfd+XZCaTFbIDgYQABoWIKLsI1SpalWqxtf2JtlLRuvys/fmj/VWqVlwrolUpogIqIDtEdiYQQjayL5NJMpOZyWT27Z7fHyeZJJBAIPfOzcx9PR+P8zh3OffO50zY3ny/3883CMMQSZIkSdKxS0t2AZIkSZI0VhiwJEmSJClODFiSJEmSFCcGLEmSJEmKEwOWJEmSJMVJRrILeKMqKirCyZMnJ7sMSZIkSSls6dKlu8MwrHzl66MuYE2ePJn6+vpklyFJkiQphQVBsOVwrztFUJIkSZLixIAlSZIkSXFiwJIkSZKkOBl1a7AkSZIkJdfAwAANDQ309vYmu5SEy8nJYcKECWRmZh7V9QYsSZIkSW9IQ0MDhYWFTJ48mSAIkl1OwoRhSEtLCw0NDUyZMuWoPuMUQUmSJElvSG9vL+Xl5WM6XAEEQUB5efkbGqkzYEmSJEl6w8Z6uNrvjd6nAUuSJEmS4sSAJUmSJGlU2bNnD9/97nff8Ocuuugi9uzZk4CKDjBgSZIkSRpVjhSwBgcHX/Nz99xzDyUlJYkqC7CLoCRJkqRR5vrrr2fDhg3MmzePzMxMcnJyKC0tZc2aNaxdu5bLL7+cbdu20dvby7XXXss111wDwOTJk6mvr6ezs5MLL7yQs88+myeffJLa2lruuusucnNzj7k2A5YkSZKkN+2rv1/Jqsa9cf3O2TVFfOXSOUd8/4YbbmDFihUsW7aMRx99lIsvvpgVK1YMt1K/+eabKSsro6enh9NPP50rr7yS8vLyQ75j3bp13HrrrfzoRz/ife97H7/5zW/40Ic+dMy1G7AkSZIkjWoLFy48ZJ+qb3/729xxxx0AbNu2jXXr1r0qYE2ZMoV58+YBMH/+fDZv3hyXWgxYkiRJkt601xppGin5+fnDjx999FEefPBBnnrqKfLy8jj33HMPu49Vdnb28OP09HR6enriUotNLo7R7s4+2rr6k12GJEmSlDIKCwvp6Og47Hvt7e2UlpaSl5fHmjVrePrpp0e0NgPWMTr7Xx/mpkfWJ7sMSZIkKWWUl5ezaNEi5s6dy3XXXXfIexdccAGDg4PMmjWL66+/njPPPHNEa3OK4DEqz8+m1REsSZIkaUT98pe/POzr2dnZ3HvvvYd9b/86q4qKClasWDH8+mc/+9m41eUI1jEqy8+ixYAlSZIkCQPWMSvLz6Kt24AlSZIkyYB1zMrzs2jpNGBJkiRJMmAds7L8LNdgSZIkSQIMWMesrCCLnoEhevqHkl2KJEmSpCQzYB2j8vwsAFq6+pJciSRJkqRkM2Ado7L8aAdopwlKkiRJI2PPnj1897vffVOf/eY3v0l3d3ecKzrAgHWMyvIzAWzVLkmSJI2Q4zlgudHwMRoewbKToCRJkjQirr/+ejZs2MC8efM4//zzqaqq4le/+hV9fX285z3v4atf/SpdXV28733vo6GhgaGhIb70pS+xa9cuGhsbWbx4MRUVFTzyyCNxr82AdYzK9q3Bci8sSZIkpaR7r4edL8X3O8edBBfecMS3b7jhBlasWMGyZcu4//77+fWvf82zzz5LGIZcdtllLFmyhObmZmpqarj77rsBaG9vp7i4mBtvvJFHHnmEioqK+Na8j1MEj1FRTgaZ6YFTBCVJkqQkuP/++7n//vs59dRTOe2001izZg3r1q3jpJNO4oEHHuBzn/scf/rTnyguLh6RehzBOkZBEFCal+UUQUmSJKWm1xhpGglhGPL5z3+eT3ziE6967/nnn+eee+7hi1/8Iueddx5f/vKXE16PI1hxUJaf5QiWJEmSNEIKCwvp6OgA4F3vehc333wznZ2dAGzfvp2mpiYaGxvJy8vjQx/6ENdddx3PP//8qz6bCI5gxUF5QRat7oMlSZIkjYjy8nIWLVrE3LlzufDCC7nqqqs466yzACgoKODnP/8569ev57rrriMtLY3MzEy+973vAXDNNddwwQUXUFNTk5AmF0EYhnH/0kRasGBBWF9fn+wyDvG/bn2Blxr28Oh1i5NdiiRJkpRwq1evZtasWckuY8Qc7n6DIFgahuGCV17rFME4KHeKoCRJkiQMWHFRmpdFR+8g/YOxZJciSZIkKYkMWHFQVuBeWJIkSUoto22p0Zv1Ru/TgBUH5fs2G251mqAkSZJSQE5ODi0tLWM+ZIVhSEtLCzk5OUf9GbsIxkGZAUuSJEkpZMKECTQ0NNDc3JzsUhIuJyeHCRMmHPX1Bqw42D+CZaMLSZIkpYLMzEymTJmS7DKOS04RjIPhEaxO98KSJEmSUpkBKw5K8rIIAqcISpIkSanOgBUH6WkBpXnuhSVJkiSlOgNWnJTlZzmCJUmSJKU4A1aclDmCJUmSJKU8A1aclOVn0WbAkiRJklKaAStOygqcIihJkiSlOgNWnJTnZ9HW3U8sNrZ3s5YkSZJ0ZAasOCnLzyIWwp6egWSXIkmSJClJDFhxMrzZcJebDUuSJEmpyoAVJ+X52QC0dLoOS5IkSUpVBqw4OTCCZcCSJEmSUpUBK07KC6KA5V5YkiRJUuoyYMVJSV4mgHthSZIkSSnMgBUn2RnpFGZnOIIlSZIkpTADVhy52bAkSZKU2gxYcVSWb8CSJEmSUpkBK47K87OcIihJkiSlMANWHEUjWG40LEmSJKUqA1YcleVn09rVTxiGyS5FkiRJUhIYsOKoPD+LgaGQjr7BZJciSZIkKQkMWHFUlh9tNtza6TosSZIkKRUZsOJoOGB1G7AkSZKkVGTAiiNHsCRJkqTUZsCKo+GAZat2SZIkKSUZsOKovCAKWO6FJUmSJKUmA1Yc5WVlkJOZ5l5YkiRJUooyYMVZeX62I1iSJElSikpYwAqCoC4IgkeCIFgVBMHKIAiuPcw1QRAE3w6CYH0QBMuDIDgtUfWMlLL8LNdgSZIkSSkqI4HfPQj8QxiGzwdBUAgsDYLggTAMVx10zYXAifuOM4Dv7TuPWgYsSZIkKXUlbAQrDMMdYRg+v+9xB7AaqH3FZe8GfhZGngZKgiAYn6iaRkK5AUuSJElKWSOyBisIgsnAqcAzr3irFth20PMGXh3CCILgmiAI6oMgqG9ubk5UmXFRasCSJEmSUlbCA1YQBAXAb4BPh2G49818RxiGPwzDcEEYhgsqKyvjW2CcleVn0d0/RO/AULJLkSRJkjTCEhqwgiDIJApXvwjD8LeHuWQ7UHfQ8wn7Xhu1yvPdC0uSJElKVYnsIhgAPwZWh2F44xEu+x3w4X3dBM8E2sMw3JGomkZC2b6A1dppwJIkSZJSTSK7CC4C/gJ4KQiCZfte+wIwESAMw+8D9wAXAeuBbuDqBNYzIsoL9o9gudmwJEmSlGoSFrDCMHwcCF7nmhD420TVkAxl+dkANrqQJEmSUtCIdBEc04YGob9r+OnwFEEDliRJkpRyDFjHIhaDGybCY/86/FJRTgaZ6YEBS5IkSUpBBqxjkZYGReOhbfPwS0EQUJrnXliSJElSKjJgHavSyYcELIimCdqmXZIkSUo9BqxjVToFWjdDGA6/VJbvCJYkSZKUigxYx6p0MvS1Q0/b8EsGLEmSJCk1GbCOVenk6HzQNMHy/CxaOt0HS5IkSUo1BqxjVTYlOh8UsMrys9nbO8jAUCw5NUmSJElKCgPWsSqZFJ3bNg2/VFYQ7YXV5jRBSZIkKaUYsI5VdgHkV75qiiBgJ0FJkiQpxRiw4uEVrdrL8h3BkiRJklKRASse9rdq38cRLEmSJCk1GbDioXQy7G2AwShQle4LWLZqlyRJklKLASseSidDGIP2bdHTvCyCwBEsSZIkKdUYsOLhFXthpacFlORm0trlXliSJElSKjFgxcPwXlgHtWrPz3KKoCRJkpRiDFjxUDAO0rNf0ao9m5ZOA5YkSZKUSgxY8ZCWBqWTXtWq3REsSZIkKbUYsOKldMqhAasgi7ZuA5YkSZKUSgxY8VI6OdoLKwyBaC+stu4BYrEwqWVJkiRJGjkGrHgpnQz9HdDdCkRTBIdiIe09A8mtS5IkSdKIMWDFyytatZft22zYvbAkSZKk1GHAipdXtGrfH7BsdCFJkiSlDgNWvJRMis6vClhuNixJkiSlCgNWvGTlQUH18BTB8vxswCmCkiRJUioxYMVT6RRo2xI9zM8EoNXNhiVJkqSUYcCKp9LJwyNY2RnpFGZn0OpeWJIkSVLKMGDFU+lkaG+AwWjdVVlBlk0uJEmSpBRiwIqn0slACHu2AVGjCwOWJEmSlDoMWPE03Kp9MwDl+Vm0uAZLkiRJShkGrHga3mw4atVemucIliRJkpRKDFjxVFANGbnDI1hVRdns7uxjcCiW3LokSZIkjQgDVjwFwSGdBKdUFDAYC9nW1pPUsiRJkiSNDANWvB0UsKZW5gOwoakzefVIkiRJGjEGrHgrnQytmyAMmVZRAMDG3QYsSZIkKRUYsOKtdDIMdEHXborzMqkoyGJjc1eyq5IkSZI0AgxY8faKVu1TKwrY0OwIliRJkpQKDFjxNtyqfTMQrcNyBEuSJElKDQaseCuZGJ337YU1tTKflq5+2rsHkliUJEmSpJFgwIq3zFwoHD88gjWtMmp0scFGF5IkSdKYZ8BKhNIpB00R3BewbNUuSZIkjXkGrEQ4aC+sutJcMtMDNu52HZYkSZI01hmwEqF0MuxthIFeMtLTmFSez0Y7CUqSJEljngErEcqmACHs2QrA1Ip8NthJUJIkSRrzDFiJ8KpW7QVsaelicCiWtJIkSZIkJZ4BKxEOsxfWwFBIQ1tP0kqSJEmSlHgGrETIr4TM/OG9sPa3at9oq3ZJkiRpTDNgJUIQHNJJcFplPgAbmlyHJUmSJI1lBqxEOShgleRlUZaf5QiWJEmSNMYZsBJlf8AKQyAaxbKToCRJkjS2GbASpWwKDHRDZxMAUysK3AtLkiRJGuMMWIlymE6Cuzv7ae8ZSFpJkiRJkhLLgJUoh9kLC3AUS5IkSRrDDFiJUjIRCA5q1R51EtzoOixJkiRpzDJgJUpGNhTVDo9g1ZXlkZEWsMERLEmSJGnMMmAl0kGt2jPT05hYnucIliRJkjSGGbASqXQytG4afjqtssC9sCRJkqQxzICVSBUnQOdO6G0Hok6Cm3d3MxQLk1yYJEmSpEQwYCVS1ezo3LQagGkVBfQPxWho605iUZIkSZISxYCVSPsD1q6VQDSCBXYSlCRJksYqA1YiFU+A7KIDI1j79sKyk6AkSZI0NhmwEikIoGoWNK0CoDQ/i9K8TDY4giVJkiSNSQasRKuaHU0RDKPGFlMrC9joCJYkSZI0JhmwEq1qNvTugY6dAEyrzGfjbkewJEmSpLHIgJVo1fs7Ce5vdFFAc0cfe3sHkliUJEmSpEQwYCXaK1q1T62wk6AkSZI0VhmwEi2vDArGwa6o0cXUfZ0EXYclSZIkjT0GrJFQPXt4iuCk8jwy0gJbtUuSJEljkAFrJFTNhuaXITZEZnoaE8vynCIoSZIkjUEGrJFQNRsGe6F1EwBTK/MNWJIkSdIYZMAaCa/oJDitsoBNLV0MxcIkFiVJkiQp3hIWsIIguDkIgqYgCFYc4f3iIAh+HwTBi0EQrAyC4OpE1ZJ0FTOA4EAnwcp8+gdjbG/rSW5dkiRJkuIqkSNYtwAXvMb7fwusCsPwFOBc4BtBEGQlsJ7kycqDsqmw68BeWAAbdtvoQpIkSRpLEhawwjBcArS+1iVAYRAEAVCw79rBRNWTdFWzoClq1T5tuFW767AkSZKksSSZa7C+A8wCGoGXgGvDMIwlsZ7Eqp4DrRthoIey/CxK8jJt1S5JkiSNMckMWO8ClgE1wDzgO0EQFB3uwiAIrgmCoD4Igvrm5uaRrDF+qmZDGIvatQNTK/LdbFiSJEkaY5IZsK4GfhtG1gObgJmHuzAMwx+GYbggDMMFlZWVI1pk3FTt7yS4v9FFgVMEJUmSpDEmmQFrK3AeQBAE1cAMYGMS60mssqmQnn1Iq/amjj46egeSXJgkSZKkeMlI1BcHQXArUXfAiiAIGoCvAJkAYRh+H/gacEsQBC8BAfC5MAx3J6qepEvPgMrpsCtqdDG1Mh+IGl2cUleSzMokSZIkxUnCAlYYhh94nfcbgXcm6ucfl6rmwKYlwIFOguuaOg1YkiRJ0hiRzCmCqad6NnQ0Qk8bUyryyclMY1Xj3mRXJUmSJClODFgjaX+ji12rSE8LmDW+iJWN7cmtSZIkSVLcGLBG0nAnwWgd1pyaIlY17iUWC5NYlCRJkqR4MWCNpKIayCkeDlhza4rp6BtkW1t3kguTJEmSFA8GrJEUBNEo1r69sObUFAOwYrvrsCRJkqSxwIA10qpmR63aw5Dp4wrISAtchyVJkiSNEQaskVY1C/raYe92sjPSObG6kBV2EpQkSZLGBAPWSKueE533TROcW1PEqsZ2wtBGF5IkSdJoZ8AaaVWzovOulUDUSXB3Zz9NHX1JLEqSJElSPBiwRlpuKRTWHOgkWLu/0YXrsCRJkqTRzoCVDNWzhwPWrPFFBAGsdB2WJEmSNOoZsJKhahY0r4WhQfKzM5hSnu8IliRJkjQGGLCSoWoODPVB6wYA5tQWO4IlSZIkjQEGrGSonh2d900TnFNTxPY9PbR19SexKEmSJEnHyoCVDBXTIUiLNhwG5tZEjS5W7XAUS5IkSRrNDFjJkJkLZdMOGcECOwlKkiRJo50BK1mqZg0HrNL8LGpLcl2HJUmSJI1yBqxkqZ4DrZugvwuA2TVFrGh0BEuSJEkazQxYyVI1Gwih+WUgWoe1aXcXXX2Dya1LkiRJ0ptmwEqW6jnReedyIFqHFYaw2kYXkiRJ0qhlwEqWsqmQWwbbngNgbm3USdB1WJIkSdLoZcBKliCAuoXQ8CwA1UXZlOdnsdJ1WJIkSdKoZcBKprqFsHstdLcSBEHU6GK7I1iSJEnSaGXASqYJC6NzQz0QTRNc19RB3+BQEouSJEmS9GYZsJKp9jQI0mHbM0DU6GJgKGTdrs4kFyZJkiTpzTiqgBUEQX4QBGn7Hk8PguCyIAgyE1taCsjKh3FzhwPW3Jr9jS5chyVJkiSNRkc7grUEyAmCoBa4H/gL4JZEFZVS6s6A7c/D0CATy/IoyM5wHZYkSZI0Sh1twArCMOwGrgC+G4bhe4E5iSsrhdSdAQNd0LSStLSo0YUjWJIkSdLodNQBKwiCs4APAnfvey09MSWlmAmnR+dtUbv2OTVFrN7RwVAsTGJRkiRJkt6Mow1YnwY+D9wRhuHKIAimAo8krqwUUjIRCsYNB6y5NcX0DAyxabeNLiRJkqTRJuNoLgrD8DHgMYB9zS52h2H4qUQWljKCAOpOH95weE5tEQArtu/lhKrCZFYmSZIk6Q062i6CvwyCoCgIgnxgBbAqCILrEltaCqk7A9o2Q8cuplUWkJWR5josSZIkaRQ62imCs8Mw3AtcDtwLTCHqJKh4qDsjOjc8S2Z6GrPGFbKy0U6CkiRJ0mhztAErc9++V5cDvwvDcACwC0O8jD8F0rOG12HNrilmxfZ2wtBfsSRJkjSaHG3A+gGwGcgHlgRBMAlwiCVeMrJh/LwDjS5qi9jbO0hDW0+SC5MkSZL0RhxVwArD8NthGNaGYXhRGNkCLE5wbamlbiE0vgCD/cypKQZwHZYkSZI0yhxtk4viIAhuDIKgft/xDaLRLMVL3UIY6oOdy5k5rpD0tMB1WJIkSdIoc7RTBG8GOoD37Tv2Aj9JVFEpacLC6LztWXIy0zmhsoAXGxzBkiRJkkaTow1Y08Iw/EoYhhv3HV8FpiaysJRTNB6KJ8K2ZwA4a1o5z2xsobt/MMmFSZIkSTpaRxuweoIgOHv/kyAIFgF2YIi3uoXDjS7On11N32CMx9ftTnJRkiRJko7W0QasvwZuCoJgcxAEm4HvAJ9IWFWpqm4hdDRCewMLp5RRmJPBA6t2JbsqSZIkSUfpaLsIvhiG4SnAycDJYRieCrw9oZWlorr967CeITM9jcUzqnh4TRNDMffDkiRJkkaDox3BAiAMw71hGO5vbff3CagntVXPhcw82PYcEE0TbOnq54WtbUkuTJIkSdLReEMB6xWCuFWhSHom1Jw23OjinBmVZKYHThOUJEmSRoljCVjOW0uEuoWwczkM9FCUk8mZU8sNWJIkSdIo8ZoBKwiCjiAI9h7m6ABqRqjG1FK3EGKD0PgCEE0T3Li7iw3NnUkuTJIkSdLrec2AFYZhYRiGRYc5CsMwzBipIlPKhAONLgDeMasawFEsSZIkaRQ4limCSoT8ciibNtzooqYklzk1RQYsSZIkaRQwYB2P6s6IRrDCaJnb+bOreX5rG80dfUkuTJIkSdJrMWAdj+oWQvduaNsERAErDOGRNU1JLkySJEnSazFgHY+GNxx+FoDZ44uoLcnlfqcJSpIkScc1A9bxqHImZBcNN7oIgoB3zKri8fXN9PQPJbk4SZIkSUdiwDoepaXDlLfBi7dB81oAzp89jt6BGI+v353k4iRJkiQdiQHreHXhv0FmDtz+Eejv5oypZRTmZPDAqp3JrkySJEnSERiwjlfFtXDFj6BpNdzzWTLT0zh3RhUPrW5iKBYmuzpJkiRJh2HAOp6dcB6c84+w7Bfwws85f3Y1LV39LNvWluzKJEmSJB2GAet4d87nYMo5cPc/8PbSJjLSArsJSpIkSccpA9bxLi0drvwvyCmh4K6PsXhyDg8YsCRJkqTjkgFrNCiogj+7GVo38oXY99nY3MmG5s5kVyVJkiTpFQxYo8XkRfD2LzFl5318KP1BHnQUS5IkSTruGLBGk0WfhhPfxVcy/5uNL/4p2dVIkiRJegUD1miSlgbv+T492RX8Xcu/sKu5JdkVSZIkSTqIAWu0ySuj5+KbqAuaWXrXt5NdjSRJkqSDGLBGoeqT38GWvLnM2XYr21ttdiFJkiQdLwxYo1TB4muZFOzi4bt+muxSJEmSJO1jwBqlyk+7gras8czY9DO2tnQnuxxJkiRJGLBGr/QMMt7yNyxMW8Mdd/8u2dVIkiRJwoA1qhWeeTW96flMWfdTNrrxsCRJkpR0BqzRLKeI2Kkf5qK0p7nl3ieSXY0kSZKU8gxYo1ze2Z8kLQioXfszXt7ZkexyJEmSpJRmwBrtSiYyOOMSPpD+MN+7f1myq5EkSZJSmgFrDMh666coCropeflXrGxsT3Y5kiRJUspKWMAKguDmIAiagiBY8RrXnBsEwbIgCFYGQfBYomoZ8yYsYLB2IR/LvI9v3r862dVIkiRJKSuRI1i3ABcc6c0gCEqA7wKXhWE4B3hvAmsZ8zIW/R117CJt7R9Ztm1PssuRJEmSUlLCAlYYhkuA1te45Crgt2EYbt13fVOiakkJMy8hVjKJT2Tdy40PrE12NZIkSVJKSuYarOlAaRAEjwZBsDQIgg8f6cIgCK4JgqA+CIL65ubmESxxFElLJ+3Mv+E01tC+7ime2/xa2VaSJElSIiQzYGUA84GLgXcBXwqCYPrhLgzD8IdhGC4Iw3BBZWXlSNY4upz6IcLsQv429z4+/9uX6OkfSnZFkiRJUkpJZsBqAO4Lw7ArDMPdwBLglCTWM/plFxLM/yjnh0/T07yZf/7DqmRXJEmSJKWUZAasu4CzgyDICIIgDzgDsAXesVr4CQLgWxOXcOuzW/njih3JrkiSJElKGYls034r8BQwIwiChiAIPhYEwV8HQfDXAGEYrgb+CCwHngX+KwzDI7Z011EqqYP5H2V+0x1cPK6df/z1crbv6Ul2VZIkSVJKCMIwTHYNb8iCBQvC+vr6ZJdxfOvaDd8+je5xCzh90zXMqSnm1mvOJD0tSHZlkiRJ0pgQBMHSMAwXvPL1ZE4RVKLkV8A515G35SF+uGgPz25u5TsPr092VZIkSdKYZ8AaqxZ+Asqmsmj9f3DlvGq+9dBa6m3dLkmSJCWUAWusysiCd/4LNK/h/0xcSl1ZHtf+zzLauweSXZkkSZI0ZhmwxrIZF8GUt5Hzpxu46T1T2bW3ly/c8RKjbd2dJEmSNFoYsMayIIB3fR162pi7/gf8wztncPdLO/hV/bZkVyZJkiSNSQassW7cSXDah+HZH/CJOTHOPqGCL925kvtX7kx2ZZIkSdKYY8BKBW//ImTkkvbgl7npqtOYXVPE3/ziee58YXuyK5MkSZLGFANWKiiogrd9Fl6+h+Idj/Pzj5/BwsllfOZXy/jFM1uSXZ0kSZI0ZhiwUsWZfwMlk+C+L1CQAT+5+nQWz6jin+5YwQ+XbEh2dZIkSdKYYMBKFRnZ8M6vQdMqeOFn5GSm8/0Pzefik8fz9XvWcOP9L9tdUJIkSTpGGckuQCNo1mUwaRHcez00rSFr0bV8+/2nkp+VzrcfXk9n3xBfumQWQRAku1JJkiRpVDJgpZIggD+7GR7+GtT/GOpvJv3UD3LD4k+Tnz2Zm5/YRFffIF+/4iTS0wxZkiRJ0hvlFMFUUzgO3n0TfOqFqH37sl+S9p35fHnwP/nKWZncVr+NT/5iKb0DQ8muVJIkSRp1gtG27mbBggVhfX19sssYO/bugCf/E+pvhsFeNla/kw9uvYSaiSfw448soCQvK9kVSpIkScedIAiWhmG44JWvO4KV6orGwwVfh0+/BGd/mqmtf+LRoq9QsP1xrvzekzS0dSe7QkmSJGnUMGApUlAJ7/jfcM1jZBdVcUvm/+Xyjlu58qbHWdnYnuzqJEmSpFHBgKVDVU6Hjz9EMOcK/hf/w42xG/irHzzIE+t3J7sySZIk6bhnwNKrZRfAlf8FF/07b2E5v0n7PP/2k9u484Xtya5MkiRJOq4ZsHR4QQAL/4rgL/9IdUEGt2d+had+/R/ceP/L9A3aYVCSJEk6HLsI6vV1tRD79V+StulRlsemsDtrAtNnzGbC5OlQPBFK6qB4AmQXJrtSSZIkaUQcqYugGw3r9eWXk/YXv4Un/5O65ZYRwGQAACAASURBVPdS3ryWyhVPwspXjGQVjIN5H4D5V0PppOTUKkmSJCWRI1h6w3oHhrjp4bX8dslSpmS28YmTM1hU2Utaw3Ow9l4IQ5h+AZz+cZj2dkhzJqokSZLGliONYBmw9KZtaO7kS3eu4MkNLZwyoZh/ufwkTirsgPqfwPM/ha5mKJsKCz4G866CvLJklyxJkiTFhQFLCRGGIb97sZGv/WE1LV19XHzSeK4970ROLM+CVb+D5/4Ltj0NGTlw4b/C/I8mu2RJkiTpmBmwlFDtPQP84LEN3PLkZnoGhrj05Bo+dd6JnFBVADuWwwNfgk1L4M9/DjMvTna5kiRJ0jExYGlEtHb188MlG/nZU5vpHRjislOioDW1OIBbLoGm1fDRP8CEV/21KEmSJI0aBiyNqJbOvn1Bawt9g0NcPq+Wz721jOpfXQL9XfDxB6L1WZIkSdIodKSAZXs3JUR5QTafv2gWf/rcYj529hTuWbGDS3+ylrXn3wLhEPz8z6CrJdllSpIkSXFlwFJCVRRk808Xz+Z3f3c2melpXH5bE0vf8l1ob4Bb3w8DPckuUZIkSYobA5ZGxPTqQu745FuYWpnPe+8Jeeykr0PDc/Dbv4LY0Ot/gSRJkjQKGLA0YqqKcrjtmrNYPKOKjzw9ngcmfgpW/x7u+6dklyZJkiTFhQFLIyo/O4MffngBHz5rEn+19gweLrkCnvkePP5NGGUNVyRJkqRXMmBpxKWnBXz1sjl88eJZ/NWuK3gy+2x48CvRmqy9jckuT5IkSXrTDFhKiiAI+Phbp3LTBxfwsa5P8q30qxnc8CjhTWfACz93NEuSJEmjkgFLSXXB3PH8zycW8UDJlZzX/XVWDE2Eu/4Wfn4l7NmW7PIkSZKkN8SNhnVciMVCfr+8kX//42oWd/yOL2TdRmZGBunv+hrMvxqCILpwaBDat0LLRmjdAK0boXImnPZhSEtP7k1IkiQpZRxpo2EDlo4rfYND/PdTW/jtQ0/wT7HvsShtJX01C8kuKIOWDdC2GWIDBz6QkQODvTDuZLjkP2DCq/4alyRJkuLOgKVRpb17gO8+uo7up37MX6fdSSyrkFjZVEonzKKodiaUT4OyaVBQBavuhD9+Hjp2wvyPwHlfgbyyZN+CJEmSxjADlkal7Xt6+K8/beSh1U1sbe0GYHp1AYtnVLF4ZhXzJ5WSmZ4GfR3w6A3w9PcgpxjO/2eY90FIc5mhJEmS4s+ApVEtDEM27u7ikTVNPLymiec2tzIwFFKYk8F7Tq3lM++YTml+FuxaCX/4e9j2NNSdARd/A8adlOzyJUmSNMYYsDSmdPQO8MT63dy/ahd3LWukIDuDf3jndK5aOJGMAHjxVnjgS9DbDu++CU55f7JLliRJ0hhiwNKYtXZXB1/9/UqeWN/CzHGFfPnS2bxlWgV0t8LtH4FNS6Ipg2/51IFuhJIkSdIxOFLAcoGKRr3p1YX8/GNn8P0Pzaezb5CrfvQMn/zFUhr6cuCDv4Y5V8ADX4b7vgCxWLLLlSRJ0hiWkewCpHgIgoAL5o7j3BmV/GjJRm56dD0PrW7iE+dM42MXfZ/igmp4+rvQuQsu/x5kZCe7ZEmSJI1BThHUmNS4p4ev37OaPyzfQV5WOn++YALX5v6Rkie+BlPeBn/+C8gpSnaZkiRJGqWcIqiUUlOSy3euOo17r30rF8wZx38/vZXTHp7FT8d9nnDzk3DLRdG+WZIkSVIcOYKllLCjvYdbntjML5/ZyqkDS/lh9rcI8yrIvvQbpJ2w2CmDkiRJekPsIigBe3sH+J9nt/Lkkgf5t4GvUxXsoTctj/bacyldcAVZMy+A7MJXfzAWg+Y10f5aW5+Bts1wzj/CCeeN+D1IkiQp+QxY0kH6B2Pcv3wrW+vvoWr7A5xLPRXBXgbIpLnyTArnvYfC8dNg23NRqNr2HPS1Rx/Or4SMHOjYAZf9J8y7Krk3I0mSpBFnwJKOoH8wxjMbmlhb/yD5G//IooGnqUtrHn5/sHwmGZPPhLozYeIZUDoF+vbCbX8Bmx6Dt38R3vpZ99iSJElKIQYs6SiEYcjqxr288NzjrFy7jj+0jqc7rZC3nljB5afWcv7savKy9u1uMNgPv/s7WH4bzL8aLvp3SHfnA0mSpFRgwJLehNU79nLnsu38flkjje295Gam88451Vw+r5azT6wgMy2Ah/4ZHr8Rpl8Af3YzZOUnu2xJkiQlmAFLOgaxWMhzm1u5c1kj97y0g/aeASoKsrj0lBquPG0Cc7bfTnDvdVBzKnzgNiioPLovDkPoaYPWjdGxdzvMugzKpyX2hiRJknRMDFhSnPQNDvHYy83c8cJ2HlrdRP9QjBOrCvj7ieu5YM0/ERSOi9ZlAQwNwFD/vmPf474OaNt0IFT1th/6A3JLo5A28YyRvzlJkiQdFQOWlADt3QP84aVG7nh+O/Vb2jg1bR0/zbmRolj7kT8UpEHJRCib+uoD4NYPRCNZV/4YZl0yMjciSZKkN8SAJSXYlpYu7nhhOw88/zKDbQ1kZWXz1pk1XDRvInPqKgjSsyA9E9KzX7sZRtdu+OX7oPGFqHHG6R8buZuQJEnSUTFgSSMkDEOe29zGr+q3cffyHfQMDHFiVQHvW1DHe06rpaIg+/W/pL8Lfv2XsPaPUQv4t3/RNvCSJEnHEQOWlAQdvQPcvXwHv6rfxvNb95CRFvD2mVV88MxJvPWECtLSXiM0DQ3C3Z+B538G8z4Il34rGgGTJElS0hmwpCRb39TB7fUN/HppAy1d/dSV5XLVwkm8d8GEI49qhSE89q/w6P+FE94B7/0pZBeMbOGSJEl6FQOWdJzoGxzi/pW7+MUzW3h6YyuZ6QEXzB3PB8+YyBlTyggONxVw6S3wh89AfhVMWADVc2HcXBh3EpRMcvqgJEnSCDNgSceh9U0d/OKZrfxmaQN7ewc5oaqA82dXs3ByGadNKqU496ApgRsejqYL7lwBLeuBfX/vZhdB9ZwodFXNgsqZ0ZFfnpR7kiRJSgUGLOk41tM/xB+WN/Kr+m28sHUPg7GQIIAZ1YWcPrmMBZNLWTiljPHFudEH+rugaTXsfAl2rYhC166V0N9x4EvzKqKgVbUvcNWdEY14OdolSZJ0zAxY0ijR0z/EC9vaqN/cxnObW3l+Sxtd/UMAjC/OYU5NEbNriplbU8Sc2mJqinOiaYVhGO2f1bwGmtZE5+aXo3Pf3ujLSyfDrEth1ruhdj6kpSXvRiVJkkYxA5Y0Sg0OxVizs4NnN7XyYsMeVjbuZWNzJ7F9f+uW5GUyp6aIubXFnH1CBQunlJGdkX7gC/YHr/UPwerfwcbHIDYAhTXRRsazLoNJb4G09MMXIEmSpFcxYEljSHf/IKt3dLCqsZ2VjXtZ2biXl3d20D8UIz8rnbNPrODtM6tYPKOKqqKcQz/cswfW3heFrfUPwmAv5JbBtMUwdXF0Lp6QnBuTJEkaJQxY0hjX0z/Ekxt289CaJh5Z08SO9l4ATqotZvHMKi46aRwzxxUd+qH+Llj3ALx8L2x8BDp3Ra9XzIBpb4/C1qRFtoaXJEl6BQOWlELCMGTNzg4eXtPEw2uaeGFrG7EQTp5QzHsX1HHZKTWHdiiMPgRNq2DDI1HHwi1PRKNbaRlRw4zsAsgqgOzCA+fsgmhd14yLoeKEpNyrJElSMhiwpBTW0tnH715s5LbntrFmZwfZGWlcMHcc71tQx1lTy0lLO0xnwYFe2PY0bFoCXc3Q1wl9HdDfGT3u74ied7dE11fOhJmXROu6xs+zW6EkSRrTDFiSCMOQlY17ue25bdy1bDt7ewepLcnlsnk1TKssoK40lwlleYwryiH9cKHrcNobYM3dsPr30ahXGIPiOph5Mcy4EMqmQkE1ZGQn9uYkSZJGkAFL0iF6B4a4b+VObq9v4IkNuzn4HwUZaQE1JbnUleVSV5rHOdMrOW9WNVkZr9PWvasF1t4Lq/8QTTMc6jvwXm4pFIyDwuoocBVUQ24JZOZBZu5B51zIzIf0zGiK4kBvdN5/DPRAbCgKcCV1ifnlvFFDg7DpUcivgqrZkJ6R7IokSVKCGbAkHVHf4BCNe3ppaOtmW2tPdG7rYVtrN5tbutjTPUBpXibvnlfLexdMYE5N8VF8aSdsfTpqEd/ZBJ07oWNn1EijY1f0fKj/zRedmQ/v+Aqc/vHktZgf6IVlv4Anvgl7tu6rKw9qToMJC/Ydp0PhuOTUJ0mSEsaAJelNGYqF/GldM7fXN/DAql30D8WYU1PEe+dP4N3zainNz3pzXxyGUcAa6I5GpQZ6osf93dF5aAAycyDjoGP/8952uPdzsOEhqF0Al/0nVM+O742/lr4OqL8ZnropCoy1C2DRp6KaG56Ljh3Lo/3GIJoyefL7YPEX3dxZkqQxYsQDVhAENwOXAE1hGM59jetOB54C3h+G4a9f73sNWFLytHX187sXG7l96TZWbN9LVnoab5tewVnTKjhzahmzxhUdvmHGG9TeM0B3/yDji3OPfFEYwku3R0GrrwPO/gy87bOJXevV3QrPfB+e+QH07oGp58Jb/wEmv/XVTT0GemHn8ihsbVoCa/8I8z8KF/+HIUuSpDEgGQHrbUAn8LMjBawgCNKBB4Be4GYDljR6rGrcy+1Lt/Hwmia2tHQDUJybycIpZZw5tfyoA1cYhjS09VC/pZX6zW3Ub25jbVMHAB85azLXvWsG+dmvsaapazfc9wVYfhtUTIdLvw2TzoJYDNq3QfPL0LzmwHnvdig/AWrmRd0Ox8+LGnG8MvTEhqB1I+x8CXatiM6bn4CBrqhb4tl/DxPmH90vKwzhoa/C4/8B86+Gi280ZEmSNMolZYpgEASTgT+8RsD6NDAAnL7vOgOWNAo17unhmU0tPL2hlac3tQwHroLsDMrysyjIzqAwJzoKsjMoyMmgIDuTra1d1G9uo6mjb/j60yaVsmBSKc0dffz8mS3UFOfy9StO4pzpla9dxLoH4Q+fgfatUD0XWjdFYWi//CqonAFFtbB7LexaeaAJR1YhjD8Zxp8STVXc+VK0J9hAdB+kZUThbcLpcObfQNWsN/5LOjhkLfhLuOgbhixJkkax4y5gBUFQC/wSWAzczGsErCAIrgGuAZg4ceL8LVu2JKpkSXGwP3At27qH9p4BOnoH6egbpLN3kM6+6OjoHaCqMIcFk6NANX9SGTPGFR7SHn7pllb+8dfL2dDcxRWn1fKli2e/9pqvvk5Y8m9RQKqYEQWqypnROa/s0GuHBqBpNex4EXYsi847X4rWeI07KQpp4+ZGjytnxmfqYRjCg/87aoqRzJA1NBBNWcwugqnnjPzPlyRpDDgeA9btwDfCMHw6CIJbcARL0mH0Dgxx0yPr+d6jGyjJy+Srl83lopPGESRiI+NYLFpLlchNksMQHvwKPPEtWPAxuPgbI7cpc3sDLP0pPP+zqItjkA7v+ynMunRkfr4kSWPIkQJWMjdrWQD8z77/SKoALgqCYDAMwzuTWJOk40xOZjr/8M4ZXDh3PJ/7zXL+9pfP887Z1Xxy8QmcXFscl6Yaw0ZiNCkI4B1fjYLWk9+OXktkyIrFYOPD8NzN0R5lYQgnng+n/XsU8m6/Gj7wP3DiOxLz81+pvQEKa5weKUkas5IWsMIwnLL/8UEjWIYrSYc1u6aIOz75Fn78+CZufGAt96/aRWVhNu+YVcV5M6tZdEIFuVmvvR/WUCyks2+Q4tzMEar6CIIAzv9nIIQn/xNig3DK+yGvPDpyS499b6/WjbDqrmjEqm0T5FXAomujToalk6NrJr8VfnoJ3PZB+OCvYcpbj/HGXsezP4J7Pht1X7ziR1BQldifJ0lSEiSyi+CtwLlEo1O7gK8AmQBhGH7/FdfeglMEJR2ltq5+Hl3bxIOrmnhsbTOdfYPkZKZx9gmVvGNWFVMq8tm+p4eGtmjT5OjcQ+OeHgZjIfPqSnjPqbVccvJ4ygsS2Nb99YQhPPClKGQdIoDckgOBq/zEqGNh7QKomg3ph/l/Y2EIjS/Ay/fAmrujJh0AkxZF671mXXr4dWRdu+GWi2HPNvjwXVB3etxvE4j2DLvvCzBhYdS+PqcYrvwvmPK2xPw8SZISzI2GJY1J/YMxntnUwoOrdvHg6ia27+k55P2qwmwmlOYyoTSPCaW5ZGek88eVO1m9Yy/paQHnTK/k8lNrOX9W9euOgCVM0xroaIz22epuOfTo2h11POxpja7NzItay9eeBhMWRB0Q1/4xClZ7t0OQFoWqGRfBzIsOjFa9lo6d8JMLoasFPvr7qJtiPD3+zWjd2ex3w5U/jlrm3/5RaN0A51wf7V92rCN2kiSNMAOWpDEvDEPW7OyguaOPCaW51JTkkpN5+P9wX7NzL3e+0Mhdy7azo72X/Kx0Lpg7ntk1RWSkBaSlBWSkBaQHAelpARnpAdkZ6dSU5FBbkktZflZiGm0c/sagbTNsXwoN9bC9HnYsP9BmPjMPpr092p9r+rte3THxaOzZFoWsgW746D1QNTM+tT/2/+CRf4G5fwbv+cGB0be+Trj776P9y6acE00ZLKyOz8+UJGkEGLAk6TBisZBnNrVy5wvbueelHXT0DR7V53Iy06gpyaX2oKOmJJfa0ujxuOIcMtMT2MhhsB92vQQ9e2DSWyAz99i/s2UD/OSi6PHV90D5tDf/XWEIj3w9apt/8vvh8u++epQqDOGFn8M910F2YTRlcH/b+DCMRvDat0H79qg5xkA3nPRnUDLxzdclSVKcGLAk6XUMDMXo7htiMBZjKAwZih16dPcPsX1PtJZre1sPje3RefueXnZ39h3yXWkBVBflDAevKRX5zK0t5qTaYqqLskdu9OuNaloDt1wE6dmw4Go44TwYf+ob6/p38KbKp/4FXPqt154CuGsV3P4R2L0OJp4Fnbui6Y6Dva++NkiHuVfAWz4VbQ4tSVKSGLAkKYF6B4bY0d67L3B1s33PwY+jIBbb94/bioJsTqot4qTaYubUFnPKhBLGFeck9wYOtmM5/P5aaHw+ep5XHk1BnHZedH6tqXxhCPd/EZ76zhvbTLm/Cx74cvSzi2uheAIUTTj08WAvPPN9WHoL9HfC1MWw6FPR+ZWBtbMZtjwOm/cdfZ0w76ooNBbVvOlfjSRJ+xmwJCmJuvsHWb1jLy81tPPS9r2s2N7OuqaO4dB1Um0xF588notPGk9dWV5yi92vazdseATWPwgbHoKu5uj1cSdBQTUM9ETT9g4+93fDQBec8ddwwQ2J2d+rZw/U3xyFrc5dUT1v+RSkZx4IVM1romuzCmDimdHj9Q9FTUBmXQKn/xVMPnvkNnmWJI05BixJOs709A+xeudentvUyj0rdvLitj0AnDyhmItPGs9Fx1PYisWiNV/rH4xCV39X1FwjMzc6svL3Pc6DyhnR1MBEh5fBPlj+q2jD5t1ro9f2B6rJZ0f7fI0/JQpeAK2boP7H8Px/Q+8eqJwFCz8OJ/85pGdFjT72bI4aigwfW6L9uk5+P8y8GLKO4s9j10p4/mew8g6onQ8X/qvrxiRpDDJgSdJxbltrN/eu2MHdy3fwYkM7AKdMKOb82dW8bXolc2uKSUtzxOVVYrFoOmBm3qGB6kj6u2HFb+C5H8GOF6P1ZkP9wEH/PkzPhtJJUDIJdr8Me7ZGLfHnXB5NNZx41qEBsq8DVvwWnv9p1O0xPSuaTrlpCRDA4i9Eo3qH28NMkjQqGbAkaRTZ1trNPS/t4J6XDoSt0rxMzj6xkredWMHbpldSXXQcrdsajcIwanu/8reQXRQFqtLJ0VEw7sDasVgMtjwBL94KK++MpkCWToZTPgB1C6ORqhW/jdaFVc6E0z4SjYrll0fB7O7Pwrr7ovB36beg5tQk3rQkKV4MWJI0Su3u7OPxdbtZsraZJet2D3csnDmukFMmlBAEMBgLGRyKMbiv4+FgLCQWC8nJSqcgK4OCnAzyszMozI7OBTkZpAcBPQND9AwM0TcwRE//0PDz9CDg0lNqOKWuJMl3f5zp64TVv4cXfwmb/gSE0cjZnCtg/kdgwumvnhoZhrDqTrj3c9E6tjP+Ghb/E2QXJOUWJEnxYcCSpDEgFgtZvXMvS9bu5k/rmlm7qyPaCDktbd95/8bIaaQF0DMwRFffIJ29g3T1D73u92emB+RkptM3GKN/MMZpE0u4etEULpg7LrH7eo1Ge7bBjmXRRsk5Ra9/fW87PPjVqEFHUS287bNRh8Sc4kOPjJzjq/lGGEYjeBsejoLkuLnJrmj02rUSckqi7piSRj0DliSluFgspKt/kK6+ITp6B4iFkJeVTnZmGrmZ6eRkpg+HqI7eAW6vb+CnT21mS0s344tz+NCZk7hq4URK87Ne9d39gzG27+lhW2s3/YMx5k0soaIge4TvcJTY9mzUBr9p1eHfT8+C3NJoumHNPBg/LzqXTjl88OrvguaXo86JTaujDZonLYITz48adLxZXS3RtMilt0DLuui1ID0agTv3+qMLlTpg+a/gzr+Jtgn4xJLoz1jSqGbAkiS9YUOxkEfWNPGTJzfxxPoWsjPSeM+ptYwvzmVbWzfbWqNj597e4Zbz+02tyOf0yWUsmFzK6ZPLmFSed/xusDzShgahbVM0qtWzJ+pq2Nt+4OjeDTtXRCMesYHoM9nF0ebKNfOioLM/UO3ZynCDjvSsqJNiT2v0vOZUOPFdcOI7o8evtyfZ/tGqpbfAqrui5h91Z8D8j0YjdUv+X/ReQTVc8PVoRGus/Zm2bIi2AJh+QbTRdjw8dRPc94Woq+SO5dH3vv/WN7aBt6TjjgFLknRM1u7q4CdPbOaOFxroHYgxriiHurJc6krzqCvbd5TmkpYWsHRLG/WbW3lucxvtPVFAqCzM5tS6EnIy0+kfjNE3OET/UGzf4+iclZFGeX4W5QXZ+85ZlOdnU16QRXVRDidWFZCRSlMVB/ujka4dy6BxWXTetTIKQuUnQNXMqN181Uyomh2NcqWlw87lsO5+WHs/NDwHhJBXASe8I9ooOjYEQwMQG4wCXGwoetz4QtTyPrsYTnl/tK6ses6hNTUshbs/E3VgnHouXPTv/P/27jw6zqtO8/j3vrWptJRWy5Is2bJsx7ETJ94SOwskTqCBkO4wLEPTmW6GZWiYGbZDN2tP032abk4zdENn4NBNs2XOAD1MmoY0BEgIISRxYhLHSex4t+VFstbSVpJqrzt/3NeyvMty2bKU53POe96qt0qvbum853U9vvf+LnXLpv8ZU8MuFHqBi/hDFUGiGx7/W1div5ADDNz2cbjtE9Nvm7Xwy7+Ap74MK34X3vwNV2nyZx+H1/wF3PrR4rVfRC47BSwRESmKZCaPMVASOv+XzkLBcqBvlGcPucD1QscQhYIlHPSIBAP+3iMc9AgHPDL5AvHRDPHRNP1jGTK5wknnK48EWd9azYbFtWxoq2HVgspX3tywvN+jdb5y9MeNxd1C0fsedvOo0qPgBV3JeC8IXsidywtArBnW3Asr33TuNb8KeTeX7NG/cotM3/Ih2PABKK05dxhJj7pg1rkVjj3v9kNHXKBbdJO/ftmt0HDd5QtcqWF46h/gma+5Hrt1/xlu+u/w+BdcMZMld7hgVFZ7YefN5+AnH4Zt/wfWvQve+HfuM1kLD7wLdj4I73zQfV4RmZUUsEREZFax1jKWybuwNZqhY3CcLe0DbDkY50DfGODmkK1bVM3GtlrqKyJ4xuB54BmDMQaDe1ywlmQ2T+qUaompTJ5M3rK0vpzVLVVc0xSbUnAU32gvPPLnbq4WAMYV6iitgWjNib0xLlj17Qbrh+bKhbBgLTSsciHr0JMwcMC9FonBoptd+KhbDrmU27JJt+WSkPWPeUEXEAMhPyyGXXgMhCFSAWX1bi5a2TzXtuNDGrMptxbaE38HyUG49i2uumPtEve6ta636aGPQ1kdvO1+aLlhan+XbBIeeDfsecj1gN3+qdPXTfv67W7/x0+4XkURmXUUsEREZM7oS6T5bfsAzxyMs6U9zt6e0Qs+RyToEQ0H8IxhYCwDuCqKK5sqWdNSxZqFVaxpqaalJqq5Y+fT8ZzrjRofcPO/Tt3nsy5INa11oappLZTPO/08I11uDtihJ1zgiu8/9+8NhF1vmj1/hcyJ95fNc9toDyS6YMmd8JrPunXKzuTYNvjBH7m2ve6v4cb3nXveWXIQvv8OOPIM3PU/4cb/cub39bwM/3wnNK+HP/zRxS1CnU25fUhr44lcTgpYIiIyZw2NZ0ikchSsxVooWEvBul4wCxjckMbScIBoOEBJMIDnnfiS3DuSYtvRIbYdGWLbkUFe6hgmmXVf2qtKQ1zTFOOapsqJ/eK6MgKeQtclN9IFw0chFIVg1AWIUKkrZR8sOXkx6ELWDfHLZ/0t44b/jfW5bbT35MfGuKGAbbedvx3JQfi398Pen7uerru+6HrPRnthrN8/r3/+vQ/DwEF489fh2jef+7wvfM9VFnzVx+DOP5/a32QsDj3boXu7K5jRvd3NmwuXwfp3w8b/qh4xkctEAUtERGSKcvkCe3tG2XZ0kB2dw+zoHGFPd4JM3g1vi4YCrGisYHlDBbGSENFwgLJw0O0jAaKhIGWRAKVhty8LuwWeS8MBIkFPPWKzUaEAT30JfvW5E8McTxUscWub3fVFWLJpaud98IOusMYf/ACuet3Jr1nrwtr+R+Hgr12Rk5HOE6/HFriewYZVrvrhzh+5YZJr7oWbPwg1bdP6qCIyNQpYIiIiFyGbL7C/d5QdncO8fGyEncdGONA3ymg6Rzp3li/cZxDwDGXhAA2VJdy6dB63LZ/HhsU1U5r7Za2lN5GmJBigsnSKRS6kuI5sgUO/cVUZjw83LPf34fILL1ufTcI3X+sWrn7/E26eWPtvXKg68Khfhh+obnUl848HqvmrTi+8ET8Am+9zPWOFnCujf+tH3PuvJMkhV3SlutW1LRSd6RaJiyrTgQAAFupJREFUTIsCloiIyCWSL1jGMzmSmTxjmTzjmRzjmTxjabcfTecYT+cmXhtL5znQN8qW9gEyuQIlIY+NbbXcdtU8brtqHovrysjmLft7R9nVNcKurhF2+vvB8SyegXWLqrl9eT2bltezorFCvWKzWfyAK3oRCLnwYfMurC1+tatiuPTOC+uNSnS7tbee+xZkRqFtkwtnNW2uiEdNmytAMl3ZlCtYEt8PzTdA9aKp/Vw+B1u/DY/9zYm12rwg1K9w8/Ka1rg5evUrp14lU2QGKWCJiIhcYZKZPM+0x3l8Tx+P7+2jvd9VR5xXEWFoPEM27/6NjgQ9ljdUsKIhxorGCgbGMjy2p4/tncMAzI9F2LS8ntuX13PL0loqSvTldNbZ+wt48suuXP2SO11wCYYv7pzJQXj2G65U/OBhJhakBiipckGrpg1ija7aY7Tar/xYfeI5uLXYenb4i1/vgP59JwqLGA+uegNs+GMXCM8W9Pc9Ar/4DPTvgdZXucqKyUG/XP/zrphIasi9NxCBFXe791zMGmvF1LvL77Gsm+mWyBVEAUtEROQKdzg+xm/29rH18CANlVFWNsVY2VhBa23ZGRdY7k2keHxPH7/e08dv9vWRSOXwDLTWlbGiMcZKf1vRGGN+LHJRvVzWWvWSzWa5NAwecnO6Bg66XrOBg640/mivK9pxPrFmaLgW5l/r9tWtsOvfYet3YDzuFr3e8D647u2u6AZAz054+M/ccMeaNvidz8Hyu04PYtbCYLsLW0e3wLbvunL8q//AlbqvWljkP8gUpYbhl38Jz33Trdd25/9wxURmemHs88lnXdsVCC8pBSwREZE5LJsv8PzhQTYfiE8MJ+wYTE68Xl0aYkVjjLryCKXhACUhV1Gx1N9HwwEKFgZGM8TH0vT764/FR9PEx1yVxmX15axbVM361mrWL6qhuVol7OeMbNIvqz84qcT+oJvLVb8C5l9zokfrtJ9NwY5/hS3/CN0vuXlka/7QLUK99TtuPbLbPgk3vHfqvXKjffDk37seOHCLNb/qYxdeIdFa16bdD8Hen4EJwIb3uwqP5xuGuPun8NOPuZL+N7wX+vZA++NuOOPdX4Km1VNrQzYFwciFz8+bDmthz8/g4c/AcAfc9nG45SMacnmJKGCJiIi8woyksuzuSkzM49rdnWBoPEMym2c84xZdzhVO/x5QVRqitixMbXmEuvIwtWURSiMBdh4bYduRIUbTOcANZVy/qJp1i6pZs7CalY0xouGL/5/9dC5PXyJNbyJNyPNY2RRTWfzZwFrX+7Tln2DXg+75De+F2z85/Tlfwx3w+BfcMMdgxA1FvPlD5z5fLgOHn3Shas/PYKQDMG4eWmrIzR+LLXBBa907XSCcLNEND/2p+wzzr4Xfuw8WrHOfZ/sD8ItPw3i/WxNt02egJHZ6GwbaYfdPYNdP3N+kaiEs+x1Y9lo3RDJcevb2T+7NGzri5uE1Xn/+gNazE37xKVdxsu4qqF0Ge34KDdfBm77meh2lqBSwRERE5DTZfIHxTJ6Uv+5XTVmY0BmGIx6XL1j29iR47vAgWw8NsPXIIEcHXE9ZwDMsqy9n1YJKrmuuZFVzFVc3VExUSBzP5OgdccGpN5GidyRNTyJF3+RjiTRD49mTfmdFSZCNbbXcsqSWm5fWsay+XD1nV7pEj5unFWsqzvniB+DXn3cBB+vK0UfKIVzh78vdsMRAyFV6TA+7tdOW3AFX3wVXvd4NlysU3HDFzfe5ao3hCheyNn4AKprg+fvhkc9CPu2GJt78wdN7f5JD8Ku/gme/CeXz4fWfh2v+g1uTbPdPXbDq2eHeO/9a14b+fa73Kzvu5pi13uIC19LXQrTKhalOf8Huzq2u93Cyuqtg1dvcOmy1S05+bXzAFQ557luut/D2T8EN73Ht3vlj1wuXHIRX/YnrBbzYuX0yQQFLRERELonekRQvdgyzvWOIlzqH2d4xTHwsA0DQMzRVRRkcy5Dwe74mCwUM9RUlzKuIUF8RoT4Wob6iZOJxIpXj6QNxnjrQPxHk5lVEuHlJLTe01hAOeKRzedK5gtuyJx7XxyJsWFzLdc2V5wyNMov0vOwWfE4nID3qqiSmE/5+1M0la1oNy98Ibbefu6fo2Avw9Fdgxw9d71BNm1u0efGr4e4vnx5kTtW5FX7yUeh60RUFSQ4ABhZuhKvvhqvfCDWLT7w/m4Ijm2HfL2H/I+53TWY8N49twVrXY7ZgHVQ0uNC2/QHXKwfu+Kq3wYrfdT1kv/68+xusfzds+vTpvXvjA/CzT8D2H7jAd89Xpz688UIUCm6x7WiV6218BVDAEhERkcvCWsux4RTbO4bY3jnMkYEktWXhk8LT/JjbV5WGptwbdXRgnM0H+tl8IM7mA3H6EunT3mMMlAQDhAKGkZQLdNFQgHWLqtmwuIYNbbVc31JJJOh61cbSOToGk3QMjk/sjw2lCHiGqtIQlVG3xaInHpf6wyAnf4U6/jDoGZbWl09pXTO5QgwddfPH2h93wwZX3zv1+VKFvOvJOrLZBbrld0F5/dR+dvAQ7P8lZMZdqGpc7Xrjzma4w4XB7f/PzSs7rm2T60WrX3Hu37f7IRcIx/rglg+7IBmMQCDstuOPgxEX9vIZVyxjYu8/zo7BcKdrz3AHDB/1t04oZN05GlbBgvXQ7G/Viy/PHLTLTAFLRERE5gxrLZ1DSYwxRIKev7lgdTywxUfT/LZ9gC3tAzxzMM7u7gTgyt4vriujN5FmwO9pOy4S9GiqipIvWIaTWUZSWS70q1LQM6xsirF2YTVrFlaxdmH1OQuCJDN54mNpkpk8VaVhqktDZ6waKTKhb4/r2apfCVe9burhJTkIP/80vPi9i2+D8aCiESqbobIFqlrcMMuRDuh4zpXez46795bWup63Rbe43rfKBRf/+68AClgiIiLyijY0npkIXO39YzRUltBcHaW5upTm6igt1aXUlYdPCkKFgiWRzjGSzDLsb8lMfuL77MQe9yCZzbO9c5htRwZ58egwSX9uW115hLULq6goCTEw5iozxkczDIxlJt5znDFQFQ1RU+YKjNSUhamrCHPLkjruWFE/0fsmMm39+2Cs3w2pzGdcGf+JfRpswc0VC4TdXK5A6MTjYNTNrYs1nbs6YT4Hfbtc2Op4zs0x69vtglnbJlhzrxvKGSo5+znGB+DQE65wR2kt3PFnRf9TXAwFLBEREZHLKJcvsLs7wbajQ2w7PMi2o0Oks3lqysPUlEWoKwtTUxampjxMbVmYaDjI0LgLXvEx17t2PIR1j6RIpHJUlYZ40+oFvHVdM9c0xVTsQ2aXgYPwwvfghe+7nq6SStejtfpeaFrjAt+RZ1ygOvhrN78N64qYXPsWV9HxCqKAJSIiIjJL5QuWJ/b18cDWDh7e2UMmV+Dqhgreuq6Ze1YvYF7FK6OogMwRhbyb87btu67qYi4FVYtcifx8GrwgNN/o5rW13e7mqF2Ba3kpYImIiIjMAcPjWf79pWM8sLWDF44OEfQMV82vwAL5QoFcwZKftAH+cMhSWvwhkS01bt9UVaIhhzKzkkNuoeq9v4C6ZS5QLbzp3AU/rhAKWCIiIiJzzL6eBA8838G+nlECniHoGTx/f/x5wULXcJKjA0mODSVPWlzaGJhf4eaitdScmIt2fG5afSxCOODhTWGhZ2stuYIlmy+QzVtKQp7Cm8xpClgiIiIir3D5gqV7JEXHwDhHB5McHRinc8jtOwaTdA0nKZzhq6ExTIS2kOcRCBgCxpC3lmzOBapMvnDSzwQ8Q2ttKVc3xFjeUMFV8yu4uqGChTWlUwpsk1lrGcvkGU3lCHhGQyLlinC2gBWcicaIiIiIyOUX8AwLqqIsqIqy4QyvZ/MFuodTHPXXBetLpMnl7UlDD3MFSy7vngc9QyjgEQx4hAPucSjoEfQMQ+NZ9vQk2N45zE+3d038jmgowOK6MiIhD8+4oGYMeMbgea4iYzqXJ5HK+VuW0XTupOC3qLaUm9pq2ehvDZXnqEQncpkpYImIiIgIAKGAR0tNKS01pUU971g6x77eUfZ0j7Cne5T2/lFyBUvBWgoFKFgX3rJ5dywSDLCwppTykiCxkhDlkSAVJUHKS4IkM3meOTjAQ9u7+JdnjwKwuK6MjW013Li4hspoaOKcBet6vwoWLO53ZHKFiZCYzbshjceHTS6tL2dlY+yc65aJnI+GCIqIiIjIrJMvWHZ1jfDMwTjPHIyzpX2ARCpXlHNXlARZ0RhjZWOMlU1uX1UawtoTwa1g7UR4C3qG+lgJ5RH1XbySaA6WiIiIiMxZ+YJlX2+CdLaAd4Zhh54BzzOEAx7BgCHoeSceBwy5vGVvT4KdXSPsPDbCrq4RdnUlTlsI+lzKwgHmx0qoj0WYHytxjysiRMMBQgGPSND9zlDAIxx0WzKbp3ckRc9Imh5/35tI0TOSYiyd54bWajZdXc+m5fXn7VkcGs+wpX2Apw/EiY9luLG1mpuX1tFWV6YeuUtAAUtERERE5ALkC5bD8TF2dSUYy+RcYPOD20SAM4ZMPk/vSNqFpETqpMCUzhXO/4t81aWhiWA2PxYhGPB4an8/h+PjACyZV8am5fVsurqe9a3VpHMFfntwgKcPxnn6QJxd3SNYC5GgR1VpiJ6RNAANsRJuXlrLLUvquHlpLY2V0Uvy93qlUcASEREREbmMrLWMpHKksnkyuQKZfMHt/cfZXIFw0GN+rIR5FRFKQmcua9/eP8Zju3t5bE8vW9oHyOQKREMB0rk8BQvhoMfahVXc1FbHTUtqub6lknDA43B8nKcO9LN5f5zNB/oZHM8C0FpbSm25K8EfDnonetf8Hrb5sQjXt1RxXXPVlCs2Hq9QWVsWPuvnmGsUsEREREREZrnxTI6nD8R5Yl8/sWiIm9pqWbOw6ryhplCw7OoeYfP+OM8eGmA0nSN7PPDlLZlc3pXbzxXoG01PLFK9oCrK6pYqrm+p5PrmKtrmldM5lORg3ygH+8Y42O/27f1jpP3AeGNrDa9aVsety+pY0RC74LL8s4UCloiIiIiInNd4JsfLx0Z48egQL/hbx2DytPcFPMPCmlLa6spom1fGotoy2vvHeHJfP3t6EgDUloW5dVkdty6tY1VzJeB6uwoFyBUKfgVJ19vXWBllQXWUwCwJZFoHS0REREREzqs0HOSG1hpuaK2ZONY/mualjiEO9Y+zoDrKknllLKwpIxz0zniOnpEUT+7r58n9/Tyxr58fv3BsSr87FDC0VJfSWlfGotpSFte54LZkXhnN1cVdPuBSUQ+WiIiIiIhcMtZadncn2N87SsAzbjNu73mGoGewFjqHxjkUH+dQ/xiH4uMcjo8xnnFVHG9sreEH779phj/JydSDJSIiIiIil50xhhWNMVY0xi7o56y19CXStPePzaoy8wpYIiIiIiJyxTHGLeBcHyuZ6aZckDMPmhQREREREZELpoAlIiIiIiJSJApYIiIiIiIiRaKAJSIiIiIiUiQKWCIiIiIiIkWigCUiIiIiIlIkClgiIiIiIiJFooAlIiIiIiJSJApYIiIiIiIiRaKAJSIiIiIiUiQKWCIiIiIiIkWigCUiIiIiIlIkClgiIiIiIiJFooAlIiIiIiJSJApYIiIiIiIiRWKstTPdhgtijOkDDs90O05RB/TPdCNkztD1JMWk60mKSdeTFIuuJSmmmbqeFllr5516cNYFrCuRMeY5a+36mW6HzA26nqSYdD1JMel6kmLRtSTFdKVdTxoiKCIiIiIiUiQKWCIiIiIiIkWigFUcX5/pBsicoutJiknXkxSTricpFl1LUkxX1PWkOVgiIiIiIiJFoh4sERERERGRIlHAEhERERERKRIFrItgjHm9MWaPMWa/MeaTM90emV2MMS3GmMeMMTuNMS8bYz7sH68xxjxijNnn76tnuq0yexhjAsaYbcaYn/jPFxtjtvj3qf9rjAnPdBtldjDGVBljHjDG7DbG7DLG3KT7k0yXMeaj/r91O4wx3zfGlOj+JFNljPmWMabXGLNj0rEz3o+Mc59/Xb1kjFl7udurgDVNxpgA8FXgDcBK4B3GmJUz2yqZZXLAx6y1K4GNwH/zr6FPAo9aa5cBj/rPRabqw8CuSc//FviStXYpMAi8Z0ZaJbPRPwA/t9ZeDVyPu650f5ILZoxZAHwIWG+tvRYIAL+P7k8ydd8BXn/KsbPdj94ALPO39wFfu0xtnKCANX03AvuttQettRngX4B7ZrhNMotYa7ustc/7jxO4Ly8LcNfR/f7b7gfeNDMtlNnGGNMMvBH4hv/cAHcAD/hv0fUkU2KMqQReDXwTwFqbsdYOofuTTF8QiBpjgkAp0IXuTzJF1trfAAOnHD7b/ege4H9b5xmgyhjTeHla6ihgTd8C4Oik5x3+MZELZoxpBdYAW4D51tou/6VuYP4MNUtmny8DHwcK/vNaYMham/Of6z4lU7UY6AO+7Q85/YYxpgzdn2QarLWdwBeBI7hgNQxsRfcnuThnux/N+Hd0BSyRGWaMKQf+FfiItXZk8mvWraOgtRTkvIwxdwO91tqtM90WmROCwFrga9baNcAYpwwH1P1JpsqfG3MPLrg3AWWcPtxLZNqutPuRAtb0dQItk543+8dEpswYE8KFq+9aa3/oH+453pXt73tnqn0yq9wC/J4x5hBuyPIduDk0Vf6QHNB9SqauA+iw1m7xnz+AC1y6P8l0vAZot9b2WWuzwA9x9yzdn+RinO1+NOPf0RWwpu9ZYJlfASeMm6z54Ay3SWYRf37MN4Fd1tq/n/TSg8A7/cfvBH58udsms4+19lPW2mZrbSvufvQra+29wGPAW/236XqSKbHWdgNHjTHL/UN3AjvR/Umm5wiw0RhT6v/bd/x60v1JLsbZ7kcPAn/kVxPcCAxPGkp4WRjXoybTYYy5CzfnIQB8y1r71zPcJJlFjDG3Ak8A2zkxZ+bTuHlYPwAWAoeB/2itPXVip8hZGWNuB/7EWnu3MaYN16NVA2wD/pO1Nj2T7ZPZwRizGlcwJQwcBN6F+49Z3Z/kghlj/hJ4O66C7jbgvbh5Mbo/yXkZY74P3A7UAT3AZ4EfcYb7kR/iv4IbhjoOvMta+9xlba8CloiIiIiISHFoiKCIiIiIiEiRKGCJiIiIiIgUiQKWiIiIiIhIkShgiYiIiIiIFIkCloiIiIiISJEoYImIyKxljMkbY16YtH2yiOduNcbsKNb5RETklSF4/reIiIhcsZLW2tUz3QgREZHj1IMlIiJzjjHmkDHmC8aY7caY3xpjlvrHW40xvzLGvGSMedQYs9A/Pt8Y82/GmBf97Wb/VAFjzD8bY142xjxsjInO2IcSEZFZQQFLRERms+gpQwTfPum1YWvtKuArwJf9Y/8LuN9aex3wXeA+//h9wOPW2uuBtcDL/vFlwFettdcAQ8BbLvHnERGRWc5Ya2e6DSIiItNijBm11paf4fgh4A5r7UFjTAjottbWGmP6gUZrbdY/3mWtrTPG9AHN1tr0pHO0Ao9Ya5f5zz8BhKy1n7v0n0xERGYr9WCJiMhcZc/y+EKkJz3Oo7nLIiJyHgpYIiIyV7190v5p//Fm4Pf9x/cCT/iPHwU+AGCMCRhjKi9XI0VEZG7R/8SJiMhsFjXGvDDp+c+ttcdLtVcbY17C9UK9wz/2QeDbxpg/BfqAd/nHPwx83RjzHlxP1QeArkveehERmXM0B0tEROYcfw7Wemtt/0y3RUREXlk0RFBERERERKRI1IMlIiIiIiJSJOrBEhERERERKRIFLBERERERkSJRwBIRERERESkSBSwREREREZEiUcASEREREREpkv8P5c7xKcqaoWEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 864x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light",
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(12, 6))\n",
    "plt.plot(range(len(train_l)), train_l, label=\"train\")\n",
    "plt.plot(range(len(test_l)), test_l, label=\"test\")\n",
    "plt.xlabel(\"Epoch\")\n",
    "plt.ylabel(\"Loss\")\n",
    "plt.legend()\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "miUxg0bDQuvs"
   },
   "source": [
    "И, наконец, посчитаем метрики"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "UXSOJFI8Quvt",
    "outputId": "cefd5ec6-b8de-4199-dba4-d546bb627f83"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overall accuracy 0.5336\n",
      "Precision [0.57644824 0.62571429 0.4488568  0.38205645 0.46306505 0.47502904\n",
      " 0.52288984 0.61395856 0.667789   0.54727273]\n",
      "Recall [0.607 0.657 0.373 0.379 0.42  0.409 0.731 0.563 0.595 0.602]\n",
      "Mean Precision 0.5323079996649001\n",
      "Mean Recall 0.5336000000000001\n"
     ]
    }
   ],
   "source": [
    "true_positive = np.zeros(10)\n",
    "true_negative = np.zeros(10)\n",
    "false_positive = np.zeros(10)\n",
    "false_negative = np.zeros(10)\n",
    "accuracy = 0\n",
    "ctn = 0\n",
    "for X, y in iter(test_loader):\n",
    "    X = X.to(device)\n",
    "    y = y.to(device)\n",
    "    with torch.no_grad():\n",
    "        y_pred = model(X).max(dim=1)[1]\n",
    "    for i in range(10):\n",
    "        for pred, real in zip(y_pred, y):\n",
    "            if real == i:\n",
    "                if pred == real:\n",
    "                    true_positive[i] += 1\n",
    "                else:\n",
    "                    false_negative[i] += 1\n",
    "            else:\n",
    "                if pred == i:\n",
    "                    false_positive[i] += 1\n",
    "                else:\n",
    "                    true_negative[i] += 1\n",
    "            \n",
    "    accuracy += torch.sum(y_pred == y).item()\n",
    "    ctn += len(y)\n",
    "print(\"Overall accuracy\", accuracy / ctn)\n",
    "print(\"Precision\", true_positive / (true_positive + false_positive))\n",
    "print(\"Recall\", true_positive / (true_positive + false_negative))\n",
    "print(\"Mean Precision\", np.mean(true_positive / (true_positive + false_positive)))\n",
    "print(\"Mean Recall\", np.mean(true_positive / (true_positive + false_negative)))"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "hw05_task.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}