notebooks/rnn_ner/rnn_torch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Дисклеймер\n",
    "Эту тетрадку нужно запускать в колабе или в vast.ai. Не мучатесь с установкой библиотек и с обучением на cpu."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading package lists... Done\n",
      "Building dependency tree       \n",
      "Reading state information... Done\n",
      "The following additional packages will be installed:\n",
      "  libpsl5 publicsuffix\n",
      "Suggested packages:\n",
      "  zip\n",
      "The following NEW packages will be installed:\n",
      "  libpsl5 publicsuffix unzip wget\n",
      "0 upgraded, 4 newly installed, 0 to remove and 10 not upgraded.\n",
      "Need to get 623 kB of archives.\n",
      "After this operation, 1887 kB of additional disk space will be used.\n",
      "Get:1 http://archive.ubuntu.com/ubuntu bionic/main amd64 libpsl5 amd64 0.19.1-5build1 [41.8 kB]\n",
      "Get:2 http://archive.ubuntu.com/ubuntu bionic/main amd64 publicsuffix all 20180223.1310-1 [97.6 kB]\n",
      "Get:3 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 wget amd64 1.19.4-1ubuntu2.2 [316 kB]\n",
      "Get:4 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 unzip amd64 6.0-21ubuntu1.1 [168 kB]\n",
      "Fetched 623 kB in 0s (1951 kB/s)\n",
      "debconf: delaying package configuration, since apt-utils is not installed\n",
      "Selecting previously unselected package libpsl5:amd64.\n",
      "(Reading database ... 6773 files and directories currently installed.)\n",
      "Preparing to unpack .../libpsl5_0.19.1-5build1_amd64.deb ...\n",
      "Unpacking libpsl5:amd64 (0.19.1-5build1) ...\n",
      "Selecting previously unselected package publicsuffix.\n",
      "Preparing to unpack .../publicsuffix_20180223.1310-1_all.deb ...\n",
      "Unpacking publicsuffix (20180223.1310-1) ...\n",
      "Selecting previously unselected package wget.\n",
      "Preparing to unpack .../wget_1.19.4-1ubuntu2.2_amd64.deb ...\n",
      "Unpacking wget (1.19.4-1ubuntu2.2) ...\n",
      "Selecting previously unselected package unzip.\n",
      "Preparing to unpack .../unzip_6.0-21ubuntu1.1_amd64.deb ...\n",
      "Unpacking unzip (6.0-21ubuntu1.1) ...\n",
      "Setting up libpsl5:amd64 (0.19.1-5build1) ...\n",
      "Setting up unzip (6.0-21ubuntu1.1) ...\n",
      "Setting up publicsuffix (20180223.1310-1) ...\n",
      "Setting up wget (1.19.4-1ubuntu2.2) ...\n",
      "Processing triggers for libc-bin (2.27-3ubuntu1.4) ...\n"
     ]
    }
   ],
   "source": [
    "!apt-get -y install unzip wget"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Archive:  data.zip\n",
      "  inflating: lenta_sample.csv        \n",
      "  inflating: __MACOSX/._lenta_sample.csv  \n"
     ]
    }
   ],
   "source": [
    "!unzip data.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting pandas\n",
      "  Downloading pandas-1.2.4-cp38-cp38-manylinux1_x86_64.whl (9.7 MB)\n",
      "\u001b[K     |████████████████████████████████| 9.7 MB 14.8 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting scikit-learn\n",
      "  Downloading scikit_learn-0.24.2-cp38-cp38-manylinux2010_x86_64.whl (24.9 MB)\n",
      "\u001b[K     |████████████████████████████████| 24.9 MB 29.5 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting matplotlib\n",
      "  Downloading matplotlib-3.4.2-cp38-cp38-manylinux1_x86_64.whl (10.3 MB)\n",
      "\u001b[K     |████████████████████████████████| 10.3 MB 24.3 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: python-dateutil>=2.7.3 in /opt/conda/lib/python3.8/site-packages (from pandas) (2.8.1)\n",
      "Requirement already satisfied: pytz>=2017.3 in /opt/conda/lib/python3.8/site-packages (from pandas) (2021.1)\n",
      "Requirement already satisfied: numpy>=1.16.5 in /opt/conda/lib/python3.8/site-packages (from pandas) (1.19.2)\n",
      "Collecting threadpoolctl>=2.0.0\n",
      "  Downloading threadpoolctl-2.1.0-py3-none-any.whl (12 kB)\n",
      "Collecting scipy>=0.19.1\n",
      "  Downloading scipy-1.6.3-cp38-cp38-manylinux1_x86_64.whl (27.2 MB)\n",
      "\u001b[K     |████████████████████████████████| 27.2 MB 21.6 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting joblib>=0.11\n",
      "  Downloading joblib-1.0.1-py3-none-any.whl (303 kB)\n",
      "\u001b[K     |████████████████████████████████| 303 kB 50.8 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting kiwisolver>=1.0.1\n",
      "  Downloading kiwisolver-1.3.1-cp38-cp38-manylinux1_x86_64.whl (1.2 MB)\n",
      "\u001b[K     |████████████████████████████████| 1.2 MB 48.5 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: pyparsing>=2.2.1 in /opt/conda/lib/python3.8/site-packages (from matplotlib) (2.4.7)\n",
      "Requirement already satisfied: pillow>=6.2.0 in /opt/conda/lib/python3.8/site-packages (from matplotlib) (8.1.2)\n",
      "Collecting cycler>=0.10\n",
      "  Downloading cycler-0.10.0-py2.py3-none-any.whl (6.5 kB)\n",
      "Requirement already satisfied: six>=1.5 in /opt/conda/lib/python3.8/site-packages (from python-dateutil>=2.7.3->pandas) (1.15.0)\n",
      "Installing collected packages: pandas, threadpoolctl, scipy, joblib, scikit-learn, kiwisolver, cycler, matplotlib\n",
      "Successfully installed cycler-0.10.0 joblib-1.0.1 kiwisolver-1.3.1 matplotlib-3.4.2 pandas-1.2.4 scikit-learn-0.24.2 scipy-1.6.3 threadpoolctl-2.1.0\n"
     ]
    }
   ],
   "source": [
    "!pip install pandas scikit-learn matplotlib"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Рекуррентные нейронные сети\n",
    "\n",
    "Рекуррентные нейронные сети (RNN) предназначены для работы с последовательностями (в том числе текстом). По устройству они похожи на цикл, где каждый шаг связан с предудущим. Работают RNN сильно дольше сверток, так как сверточные слои применяются независимо и можно параллельно вычислить все трансформации, а в RNN вычислять шаг можно только после того, как вычеслен предыдущий. Неэффективность, как правило, компенсируется точностью, поэтому RNN являются стандартным инструментом при работе с текстами."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Давайте посмотрим, как в торче использовать рекуррентные слои и как их соединять с Embedding и Linear слоями."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Для начала попробуем уже привычную классификацию новостей из ленты"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import torch.utils.data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from string import punctuation\n",
    "from sklearn.model_selection import train_test_split\n",
    "from collections import Counter\n",
    "from IPython.display import Image\n",
    "from IPython.core.display import HTML \n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv('lenta_sample.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.dropna(subset=['topic', 'text'], inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Предобработка нам тут особо не важна."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess(text):\n",
    "    tokens = text.lower().split()\n",
    "    tokens = [token.strip(punctuation) for token in tokens]\n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "В предыдущей домашке не хватало фмеры. Ее нет в керасе, но можно взять вот такую функцию (взятую со стаковерфлоу)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Предобработка точно такая же"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# словарь\n",
    "vocab = Counter()\n",
    "\n",
    "for text in data.text:\n",
    "    vocab.update(preprocess(text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "670782"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# отфильтрованный словарь\n",
    "filtered_vocab = set()\n",
    "\n",
    "for word in vocab:\n",
    "    if vocab[word] > 30:\n",
    "        filtered_vocab.add(word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "55293"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(filtered_vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# индексируем слова\n",
    "word2id = {'PAD':0}\n",
    "\n",
    "for word in filtered_vocab:\n",
    "    word2id[word] = len(word2id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "id2word = {i:word for word, i in word2id.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Dataset(torch.utils.data.Dataset):\n",
    "\n",
    "    def __init__(self, word2id, MAX_LEN, texts, targets):\n",
    "        \n",
    "        # храним сырой датасет\n",
    "        # еще лучше можно сделать, если считывать из файла при генерации\n",
    "        self.texts = [torch.LongTensor([word2id[w] for w in self.preprocess(t) if w in word2id][:MAX_LEN]) for t in texts]\n",
    "        self.texts = torch.nn.utils.rnn.pad_sequence(self.texts, batch_first=True)\n",
    "        \n",
    "        # мы не считаем словарь прямо тут, т.к. нам нужно шерить словарь между трейном и валидацией\n",
    "        self.word2id = word2id\n",
    "        self.MAX_LEN = MAX_LEN\n",
    "        self.length = len(texts)\n",
    "        # таргеты можно сразу целиком выделить \n",
    "        \n",
    "        self.target = torch.LongTensor(targets)\n",
    "\n",
    "    def __len__(self):\n",
    "        # нужно уметь возвращать длину датасета, в нашем случае это просто сделать\n",
    "        # если не загружать данные целиком, а читать из файла - это может быть сложнее\n",
    "        return self.length\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "\n",
    "        # в index будут индексы от 0 до self.length\n",
    "        # по ним мы достанем тексты, предобработаем,переведем в векторы, западим и вернем\n",
    "\n",
    "        ids = self.texts[index]\n",
    "        y = self.target[index]\n",
    "\n",
    "        return ids, y\n",
    "    \n",
    "    def preprocess(self, text):\n",
    "        tokens = text.lower().split()\n",
    "        tokens = [token.strip(punctuation) for token in tokens]\n",
    "        tokens = [token for token in tokens if token]\n",
    "        return tokens\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_LEN = 200"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "texts = data.text.values\n",
    "id2label = {i:l for i,l in enumerate(set(data.topic))}\n",
    "label2id = {l:i for i,l in id2label.items()}\n",
    "targets = [label2id[l] for l in data.topic]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_texts, valid_texts, train_targets, valid_targets = train_test_split(texts, targets, test_size=0.05)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_set = Dataset(word2id, MAX_LEN, train_texts, train_targets)\n",
    "training_generator = torch.utils.data.DataLoader(training_set, batch_size=256, shuffle=True, )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "valid_set = Dataset(word2id, MAX_LEN, valid_texts, valid_targets)\n",
    "valid_generator = torch.utils.data.DataLoader(valid_set, batch_size=256, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import f1_score\n",
    "\n",
    "def train(model, iterator, optimizer, criterion, print_every=1000):\n",
    "    \n",
    "    epoch_loss = []\n",
    "    epoch_f1 = []\n",
    "    \n",
    "    model.train()  \n",
    "\n",
    "    for i, (texts, ys) in enumerate(iterator):\n",
    "        \n",
    "        optimizer.zero_grad()   \n",
    "        predictions = model(texts.to(device)).squeeze()  \n",
    "        loss = criterion(predictions, ys.to(device))        \n",
    "        \n",
    "        \n",
    "        \n",
    "        loss.backward()\n",
    "        optimizer.step()     \n",
    "        preds = predictions.detach().to('cpu').numpy().argmax(1).tolist()\n",
    "        y_true = ys.tolist()\n",
    "        \n",
    "        epoch_loss.append(loss.item())\n",
    "        epoch_f1.append(f1_score(y_true, preds, average=\"micro\"))\n",
    "        \n",
    "        if not (i+1) % print_every:\n",
    "            print(f'Loss: {np.mean(epoch_loss)}; F1 measure: {np.mean(epoch_f1)}')\n",
    "        \n",
    "    return np.mean(epoch_f1)\n",
    "\n",
    "\n",
    "def evaluate(model, iterator, criterion):\n",
    "    \n",
    "    epoch_loss = []\n",
    "    epoch_f1 = []\n",
    "    \n",
    "    model.eval()  \n",
    "    with torch.no_grad():\n",
    "        for texts, ys in iterator:   \n",
    "\n",
    "            predictions = model(texts.to(device)).squeeze()  \n",
    "            loss = criterion(predictions, ys.to(device))        \n",
    "            preds = predictions.detach().to('cpu').numpy().argmax(1).tolist()\n",
    "            y_true = ys.tolist()\n",
    "            \n",
    "\n",
    "            epoch_loss.append(loss.item())  \n",
    "            epoch_f1.append(f1_score(y_true, preds, average=\"micro\"))\n",
    "            \n",
    "    return np.mean(epoch_f1)# / len(iterator)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Есть несколько разновидностей рекуррентных слоев - RNN (vanila rnn), GRU, LSTM. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Первый - самый простой рекуррентный слой, где на каждом шаге используется предыдущее состояние и текущий вектор, оба они умножаются на соответствующую матрицу весов и получившиеся вектора складываются в один, который передается на следующий шаг. Проблема с обычными RNN в том, что информация о предыдущих состояних очень быстро затирается и учитывать нелокальные зависимости не получается. \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "LSTM и GRU частично решают это проблему добавление дополнительных матриц весов и активаций, которые будут отвечать за то, чтобы нужная информация сохранялась на протяжении всей последовательности. GRU немного проще и поэтому немного быстрее, но это не всегда значит, что LSTM лучше по качеству. \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<img src=\"https://habrastorage.org/webt/xt/_q/nj/xt_qnjgfjengqoqd4gizkq4j_wk.png\" width=\"1000\" height=\"1000\"/>"
      ],
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# lstm cell\n",
    "Image(url=\"https://habrastorage.org/webt/xt/_q/nj/xt_qnjgfjengqoqd4gizkq4j_wk.png\",\n",
    "     width=1000, height=1000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Ссылки, чтобы разобраться подробнее:  \n",
    "    http://karpathy.github.io/2015/05/21/rnn-effectiveness/  \n",
    "    https://towardsdatascience.com/illustrated-guide-to-lstms-and-gru-s-a-step-by-step-explanation-44e9eb85bf21  \n",
    "    https://colah.github.io/posts/2015-08-Understanding-LSTMs/  \n",
    "    \n",
    "    \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Использовать RNN слой в torch достаточно просто при условии, что применяется общий паддинг (все обучающие данные до одной длины).\n",
    "Нужно просто поставить его после другого слоя, который возвращает последовательность.\n",
    "На выходе из vanila RNN  и GRU  мы получим два объекта - (output и h), а на выходе из LSTM - (output, (h, c)).\n",
    "В output будут лежать скрытые состояния (h_t) на всех шагах последовательности. А в h - последнее скрытое состояние последовательности. У LSTM еще есть последнее внутреннее состояние c. На картинке выше видно, что он используется при расчете h, но внутри сети еще и передается между последовательностями отдельно. \n",
    "\n",
    "Для классификации мы можем использовать h, так как в нем накоплена вся информация о последовательности и можно уже к этому вектору применять софтмакс, чтобы получить вероятности классов."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Clf(nn.Module):\n",
    "    \n",
    "    def __init__(self, output_dim):\n",
    "        \n",
    "        super().__init__()          \n",
    "        self.VOCAB_SIZE = len(word2id)\n",
    "        self.EMB_DIM = 50\n",
    "        self.RNN_UNITS = 128\n",
    "        \n",
    "        \n",
    "        self.embedding = nn.Embedding(self.VOCAB_SIZE, self.EMB_DIM)\n",
    "        \n",
    "        # все три типа рнн слоев применяются почти одинаково\n",
    "#         self.rnn = nn.LSTM(self.EMB_DIM, self.RNN_UNITS, batch_first=True)\n",
    "        self.rnn = nn.GRU(self.EMB_DIM, self.RNN_UNITS, batch_first=True)\n",
    "#         self.rnn = nn.RNN(self.EMB_DIM, self.RNN_UNITS, batch_first=True)\n",
    "        self.fc = nn.Linear(self.RNN_UNITS, output_dim)\n",
    "        self.act = nn.LogSoftmax(1) \n",
    "        \n",
    "        \n",
    "    def forward(self, text):\n",
    "\n",
    "        embedded = self.embedding(text)\n",
    "        # применяем rnn слой\n",
    "        rnn_output, (h, *_) = self.rnn(embedded)\n",
    "        \n",
    "        # lstm всегда будет возвращать трехмерный h\n",
    "        # (1, 256, 128) - в текущем случае\n",
    "        # от первой размерности нужно избавиться\n",
    "        # для этого нужна функция squeeze(0) где ноль индекс размерности\n",
    "        \n",
    "        # для GRU или RNN эта строчка ничего не изменит и все будет работать\n",
    "        h = h.squeeze(0)\n",
    "        \n",
    "        output = self.act(self.fc(h))        \n",
    "        return output\n",
    "\n",
    "\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')  \n",
    "model = Clf(len(label2id))\n",
    "optimizer = optim.Adam(model.parameters(), lr=0.001)\n",
    "criterion = nn.NLLLoss() \n",
    "model = model.to(device)\n",
    "criterion = criterion.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "Loss: 2.3700149035453797; F1 measure: 0.213359375\n",
      "Loss: 2.3331090700626373; F1 measure: 0.212421875\n",
      "Loss: 2.2547484000523883; F1 measure: 0.2358203125\n",
      "Loss: 2.1554035997390746; F1 measure: 0.268388671875\n",
      "Loss: 2.056787453174591; F1 measure: 0.3053515625\n",
      "Eval -  0.5508159722222222\n",
      "1\n",
      "Loss: 1.2984862744808197; F1 measure: 0.60140625\n",
      "Loss: 1.2393755167722702; F1 measure: 0.6197265625\n",
      "Loss: 1.1944325002034506; F1 measure: 0.6339192708333333\n",
      "Loss: 1.1523097196221352; F1 measure: 0.647744140625\n",
      "Loss: 1.1127120413780212; F1 measure: 0.65971875\n",
      "Eval -  0.7247150383141763\n",
      "2\n",
      "Loss: 0.8344336587190628; F1 measure: 0.7448046875\n",
      "Loss: 0.8252625700831413; F1 measure: 0.74810546875\n",
      "Loss: 0.8142835609118144; F1 measure: 0.7508984375\n",
      "Loss: 0.7989979486167431; F1 measure: 0.754453125\n",
      "Loss: 0.7850586802363396; F1 measure: 0.7583828125\n",
      "Eval -  0.7666343390804599\n",
      "3\n",
      "Loss: 0.6430188620090485; F1 measure: 0.7987109375\n",
      "Loss: 0.6335504747927189; F1 measure: 0.8025\n",
      "Loss: 0.6356003288427988; F1 measure: 0.80203125\n",
      "Loss: 0.6305458659678698; F1 measure: 0.80279296875\n",
      "Loss: 0.6242742669582367; F1 measure: 0.8043046875\n",
      "Eval -  0.7851616379310344\n",
      "4\n",
      "Loss: 0.5186176463961601; F1 measure: 0.8337890625\n",
      "Loss: 0.5181113524734974; F1 measure: 0.8353125\n",
      "Loss: 0.5190469771623611; F1 measure: 0.8349088541666667\n",
      "Loss: 0.520758700221777; F1 measure: 0.83486328125\n",
      "Loss: 0.5196268438100815; F1 measure: 0.8355234375\n",
      "Eval -  0.7912278496168582\n",
      "5\n",
      "Loss: 0.43842388421297074; F1 measure: 0.863515625\n",
      "Loss: 0.43692936316132547; F1 measure: 0.86236328125\n",
      "Loss: 0.4374125594894091; F1 measure: 0.8623567708333333\n",
      "Loss: 0.4383550577610731; F1 measure: 0.861689453125\n",
      "Loss: 0.43647893679141997; F1 measure: 0.862578125\n",
      "Eval -  0.7981764846743296\n",
      "6\n",
      "Loss: 0.359384478777647; F1 measure: 0.8884375\n",
      "Loss: 0.3586208900809288; F1 measure: 0.88927734375\n",
      "Loss: 0.36185421933730444; F1 measure: 0.8870182291666666\n",
      "Loss: 0.36424450654536483; F1 measure: 0.885869140625\n",
      "Loss: 0.364161028534174; F1 measure: 0.8855859375\n",
      "Eval -  0.7916738505747126\n",
      "7\n",
      "Loss: 0.284687140583992; F1 measure: 0.91234375\n",
      "Loss: 0.2838332697749138; F1 measure: 0.911875\n",
      "Loss: 0.28969630350669223; F1 measure: 0.9102473958333334\n",
      "Loss: 0.2930147705972195; F1 measure: 0.90921875\n",
      "Loss: 0.29528693395853045; F1 measure: 0.9085703125\n",
      "Eval -  0.7942193486590038\n",
      "8\n",
      "Loss: 0.22699383333325385; F1 measure: 0.9329296875\n",
      "Loss: 0.22894252270460128; F1 measure: 0.9323828125\n",
      "Loss: 0.23068149109681446; F1 measure: 0.9315104166666667\n",
      "Loss: 0.23242167096585034; F1 measure: 0.930302734375\n",
      "Loss: 0.23416637170314789; F1 measure: 0.9291484375\n",
      "Eval -  0.7841678639846743\n",
      "9\n",
      "Loss: 0.1661054788529873; F1 measure: 0.9523828125\n",
      "Loss: 0.1734994785115123; F1 measure: 0.949765625\n",
      "Loss: 0.17483642069001992; F1 measure: 0.9495963541666667\n",
      "Loss: 0.17668531807139515; F1 measure: 0.94849609375\n",
      "Loss: 0.178256261870265; F1 measure: 0.9478515625\n",
      "Eval -  0.7859093630268199\n"
     ]
    }
   ],
   "source": [
    "f1s = []\n",
    "f1s_eval = []\n",
    "\n",
    "for i in range(10):\n",
    "    print(i)\n",
    "    f1s.append(train(model, training_generator, optimizer, criterion, 100))\n",
    "    evl = evaluate(model, valid_generator, criterion)\n",
    "    print('Eval - ', evl)\n",
    "    f1s_eval.append(evl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAvWUlEQVR4nO3deXxddZ3/8dcne9NmadIkbdOkTaF7gRZCqSCIQFlEQUUFBMYVdFwQdfwJjqMj44zOjKOjDi7oIKgsMoBanY4tO+PQQltaoUlXuiVts7RZu6TZPr8/zm24DWmbpjm5yb3v5+NxHz3L9577uRfy/ZzzPd/z/Zq7IyIiiSsp1gGIiEhsKRGIiCQ4JQIRkQSnRCAikuCUCEREEpwSgYhIglMikIRkZveb2Tf7WXa7mV12jH2jzOwPZtZsZv81uFG+6bNKzWy/mSWH+TmSeJQIRE7N+4AiIN/d329mE8xssZntNjM3symD9UHuvtPdx7h712AdUwSUCERO1WRgk7t3Rta7gT8B18UupKOZWUqsY5DhTYlAhq1Ik8yXzOxVMztgZv9pZkVm9j9m1mpmT5nZ2Kjy15hZhZk1mdlzZjYrat98M3sl8r7fABm9PuudZrY28t4XzezMfsT3DeBrwPWRJpuPuXutu/8IWDnY39HMpkSuMlIi63lm9ovI1Uejmf0usv1iM6s2sy+bWQ3wCzNLN7N/j5TdHVlO70+MEv+UCGS4uw5YBEwH3gX8D/AVoIDg/9/bAcxsOvAwcEdk3xLgD2aWZmZpwO+AXwF5wH8RdcZuZvOB+4BPAPnAT4HFJ6oo3f3rwD8Bv4k02fxnmN+xD78CMoE5QCHwvah94wm+62TgNuBvgYXAPOAsYAHw1QHGK3FGiUCGux9GzrJ3Af8LvOTua9y9DfgtMD9S7nrgv939SXfvAL4DjALOJ6gAU4F/d/cOd3+Mo8/YbwN+6u4vuXuXuz8AHI68byj09zv2MLMJwFXAJ929MfK9no8q0g183d0Pu/sh4Cbgbnevc/d64BvALWF/MRkZ1HYow11t1PKhPtbHRJYnAjuO7HD3bjOrAoqBLmCXHz3C4o6o5cnAh8zss1Hb0iLHHAr9/Y7RSoAGd288xjHrI4nkiKN+n8jyUH0/GeZ0RSDxYjdBhQ6AmRlBZbkL2AMUR7YdURq1XAX8o7vnRr0y3f3hoQh8gKqAPDPLPcb+3sMKH/X7EHz/3SHEJSOQEoHEi0eBq83sUjNLBb5I0LzzIrAc6ARuN7NUM3svQRv5ET8DPmlm51lgtJldbWZZAwnEzDKAI/cX0iPrg8rd9xDcS/iRmY2NfK+LjvOWh4GvmlmBmY0juMn968GOS0YmJQKJC+6+EbgZ+CGwl+Cm67vcvd3d24H3Ah8GGgjuJzwR9d5VwK3AfwCNwJZI2YE6BOyPLG+IrIfhFqAj8hl1BDfKj+WbwCrgVeA14JXINhFME9OIiCQ2XRGIiCQ4JQIRkQSnRCAikuCUCEREEtyIe6Bs3LhxPmXKlFiHISIyoqxevXqvuxf0tW/EJYIpU6awatWqWIchIjKimNmOY+1T05CISIJTIhARSXBKBCIiCW7E3SPoS0dHB9XV1bS1tZ248AiWkZHBpEmTSE1NjXUoIhJH4iIRVFdXk5WVxZQpUzh6gMn44e7s27eP6upqysrKYh2OiMSRuGgaamtrIz8/P26TAICZkZ+fH/dXPSIy9OIiEQBxnQSOSITvKCJDLy6ahkRE4lF3t7Oz4SAbalrZWNPKpbMKmVucM+ifo0QwCJqamnjooYf41Kc+dVLve8c73sFDDz1Ebm5uOIGJyIjRcKCdDXtaeir9DbWtbKpp5VBHFwBmkDcmTYlguGpqauJHP/rRmxJBZ2cnKSnH/omXLFkSdmgiMsy0dXSxpW4/G2pa2bCnhY21rWyoaaW+9XBPmbzRacwcn8UNC0qYOT6LmeOzmVY0hsy0cKpsJYJBcOedd/L6668zb948UlNTycjIYOzYsWzYsIFNmzbx7ne/m6qqKtra2vjc5z7HbbfdBrwxXMb+/fu56qqreOtb38qLL75IcXExv//97xk1alSMv5mIDFR3t1PdeIj1NS1sPHKWX9PCtr0H6I7MB5aeksS0ojG8bXoBM8dnMSPyKhiTPqT3BOMuEXzjDxVU7m4Z1GPOnpjN198155j7v/3tb7Nu3TrWrl3Lc889x9VXX826det6unned9995OXlcejQIc4991yuu+468vPzjzrG5s2befjhh/nZz37GBz7wAR5//HFuvvnmQf0eIhKOxgPtwRl+pNLfUNPKptpWDra/0axTmpfJjKIsrj5jAjMnZDNjfBZT8keTnBT7TiBxlwiGgwULFhzV1/8HP/gBv/3tbwGoqqpi8+bNb0oEZWVlzJs3D4BzzjmH7du3D1W4ItJPR5p1Nta0srG2lfV7goq/LqpZZ2xmKjPHZ/OB8pKes/zpRVmMTh++1e3wjWyAjnfmPlRGjx7ds/zcc8/x1FNPsXz5cjIzM7n44ov7fBYgPT29Zzk5OZlDh8Ka71xE+qO+9TDrdjdTubuFykiFv23vAboi7TppKUlMKxzDhdPeaNaZOT6LgqyhbdYZDHGXCGIhKyuL1tbWPvc1NzczduxYMjMz2bBhAytWrBji6ETkeNydPc1trNvVzLrdLVTsambd7mZqW944yy/Ny2TG+CyumjuemeOPNOtkkpIcH49iKREMgvz8fC644ALmzp3LqFGjKCoq6tl35ZVX8pOf/IRZs2YxY8YMFi5cGMNIRRLbkX7563Y3s25XCxW7m6nY3ULDgXYAkgxOLxzDBaeNY05xDnMnZjN7YjZZGfE9vpe5e6xjOCnl5eXee2Ka9evXM2vWrBhFNLQS6buKnIqubmdr/f6eSn/drqCZp/VwJwCpycaM8VnMmZDD3OJs5hTnMGt8NqPSkmMceTjMbLW7l/e1T1cEIjLitXd2s6m2lYojlf7uZtbvaaGtoxuAjNQkZk3I5t3zi4NKf2IO04uySEuJj6adU6VEICIjSltHF+v3tBzVnr+xppWOrqB1Y0x6CnMmZnPTeZOZMzGbucU5TB03Om7a88OgRCAiw1ZrWweVu4+u9F+vf6PnztjMVOYW5/Cxt05lbnE2cyfmUJqXSdIw6Js/koSaCMzsSuD7QDLwc3f/dq/9k4H7gAKgAbjZ3avDjElEhqeOrm421rSyZmcja6qaWFvVxNb6Az37i7LTmTMxhyvnjA9u5BbnMDEnY8R11RyOQksEZpYM3AMsAqqBlWa22N0ro4p9B/iluz9gZpcA3wJuCSsmERke3J3dzW2s3dnE2qpG1uxs4rVdzRzuDNr0x41JZ15JLu+ZV8zcSTnMmZhNYVZGjKOOX2FeESwAtrj7VgAzewS4FohOBLOBL0SWnwV+F2I8IhIjBw538mp1M2uqGiOVf1PP07hpKUnMnZjNzQsnM68kl3kluUwaO0pn+kMozERQDFRFrVcD5/Uq8xfgvQTNR+8Bssws3933RRcys9uA2wBKS0tDC3iojBkzhv3798c6DJFQdHU7W+r2s7aqkbVVTazZ2cSm2taegdbKxo3mgtPHMa8kl/mlucwcn63eOzEW65vFfwP8h5l9GHgB2AV09S7k7vcC90LwHMFQBigix1ffejhS4QcV/6vVzeyP9NXPGZXKWSW5XDFnPPNKc5k3KZexo9NiHLH0FmYi2AWURK1Pimzr4e67Ca4IMLMxwHXu3hRiTKG48847KSkp4dOf/jQAf//3f09KSgrPPvssjY2NdHR08M1vfpNrr702xpGKnJq2ji4qdjezJtK8s2ZnE7uagnGxUpKMWROyec/84p6z/bJxo9XEMwKEmQhWAtPMrIwgAdwAfDC6gJmNAxrcvRu4i6AH0an5nzuh5rVTPsxRxp8BV337mLuvv/567rjjjp5E8Oijj7J06VJuv/12srOz2bt3LwsXLuSaa67RH4WMGO7O9n0He27mrq1qYv2elp7++sW5o5hXkstHLpjCvJJc5hbnkJEan0/lxrvQEoG7d5rZZ4ClBN1H73P3CjO7G1jl7ouBi4FvmZkTNA19Oqx4wjR//nzq6urYvXs39fX1jB07lvHjx/P5z3+eF154gaSkJHbt2kVtbS3jx4+PdbgifWo62M7aSLfNI6+mgx0AZKYlc9akXD5+4dTgbL8kl8Js9eKJF6HeI3D3JcCSXtu+FrX8GPDYoH7occ7cw/T+97+fxx57jJqaGq6//noefPBB6uvrWb16NampqUyZMqXP4adFYqGjq5sNe1qPOtvfujfos28G0wuzuGL2eOaX5jKvNJdphVnDYgIVCUesbxbHjeuvv55bb72VvXv38vzzz/Poo49SWFhIamoqzz77LDt27Ih1iJKg3J1dTYeCs/ydTaypamJdrz7780tzue6cScwvyeWMSTlxP9qmHE2JYJDMmTOH1tZWiouLmTBhAjfddBPvete7OOOMMygvL2fmzJmxDlESxP7Dnbxa1dTzdO6anU3s3R/02U9PSWJucQ43L5wcnO2X5FKcqz77iU6JYBC99tobN6nHjRvH8uXL+yynZwhksHR1O5vrWoPmnUgTz6a6Vo6MLj913GgumjaOeaW5zC8Zy8wJWaRq8DXpRYlAZASpa2ljTeQsf21VI69VN3MgMkF6zqhU5pXkctUZ43ue0M3NVJ99OTElApFh6lB7F+t2N/ec6a/Z2cju5qDDQUqSMXtiNtedMynSZ38sU/Iz1cQjAxI3icDd4/6PYKTNJif95+5UNRxi5faGYDyeqibW72ntGW65OHcU8yeP5aORB7XmTFSffRk8cZEIMjIy2LdvH/n5+XGbDNydffv2kZGhvtvxwN3ZtvcAL21r4KWt+3hpWwN7Imf7Y9JTOHNSDp+4aCrzS8dyVkmORt6UUMVFIpg0aRLV1dXU19fHOpRQZWRkMGnSpFiHIQPgHgzEtiJS8b+8raFn9M1xY9I4ryyf86bmsaAsT332ZcjFRSJITU2lrKws1mGI9OjudjbWtvac7b+8rYF9B9qBYIKVhVODiv+8snxOK9B4PBJbcZEIRGKtq9tZv6eFFZGKf+X2hp7hGYpzR/G26QU9Ff9k3dSVYUaJQGQAOru6Wbe7peeMf+X2BlrbgqGXS/MyWTSriPOm5nNeWR4leZkxjlbk+JQIRPqho6ubV6ubeWnbPlZsbWD19oae/vtTx43mnWdO4LyyfBaU5TExd1SMoxU5OUoEIn043NnFX6qae874V+9o5FBHUPFPKxzDe84uDm7wluVpFE4Z8ZQIRAgmXHllZyMvbW3gpW37WLOzqWdQtpnjs7j+3BLOK8vj3LI8xo1Jj3G0IoNLiUASVlXDQZ7dWMfT6+tYvnUf7Z3dJBnMjkykfl5Z0J1TwzRIvFMikITR1e2s2dnI0xvqeGZ9HRtrW4FgMvVbFk7mgtPzKZ+SR7aGYJYEo0Qgca2lrYMXNtXz9Po6nttYR+PBDlKSjHOn5PHVq2dxycxCphaMiXWYIjGlRCBxZ2v9fp7ZEDT5rNzeQGe3MzYzlbfPKOSSWYVcOK2AnFE66xc5QolARryOrm5Wbm/gmfV1PL2hjm2RKRdnFGVx60VTuWxWIfNKxmrYBpFjUCKQEanhQDvPbQwq/hc21tN6uJO05CTeclo+H7lgCm+fUagHuUT6SYlARgT3YOyep9fX8cyGOl7Z2Yg7FGSlc/WZE7hkZiEXnD6O0en6X1rkZOmvRoatto4uVmzd19Pev6vpEABnFOdw+yXTuHRWIXMn5pCkJh+RU6JEIMNKXUtbUPFvqOPPm/dyqKOLUanJvHXaOD57yem8fWYhRXqSV2RQhZoIzOxK4PtAMvBzd/92r/2lwANAbqTMne6+JMyYZHjp7nYqdrfw9IZantlQx6vVzUAwYuf7zpnEpbMKWTg1X7NxiYQotERgZsnAPcAioBpYaWaL3b0yqthXgUfd/cdmNhtYAkwJKyYZPupa2/jNy1U8/PJOdje3YQZnl47lS1fM4NJZhcwoytJQzSJDJMwrggXAFnffCmBmjwDXAtGJwIHsyHIOsDvEeCTG3J2V2xv51Yod/GndHjq6nAunjeOLl8/g4hkF5GsMH3AHJUAZYmEmgmKgKmq9GjivV5m/B5aZ2WeB0cBlfR3IzG4DbgMoLS0d9EAlXAcOd/LbNbv49YodbKhpJTsjhb96yxRuOq80/p7qdYeOg3CoCdqaoa0pshxZP9Fyx0FIy4KMHMjIjvybA+lRy2/annv09hQlVDk5sb5ZfCNwv7v/m5m9BfiVmc119+7oQu5+L3AvQHl5uccgThmALXWt/HrFTh5fXU3r4U7mTMzmn687g2vOKmZU2jBu8+/uhsMtJ1+JH6n8uzuOf/zoyntULuRNDf7NyIXUTGjfH0kikVfLbji84Y31o/883iwl4wSJI+fNr+jtaaPfuCpxh64O6GqPvHovHz7+/s7Dx3hfe6/XMfYnp8OYIsgqCv4dUxj5N/JKj7MTiRgJMxHsAkqi1idFtkX7GHAlgLsvN7MMYBxQF2JcEqLOrm6erKzlVyt28OLr+0hLTuLqMydw88LJnF2aG367f3cXHG6NvFqCf9taIsstkeXWPpab3ziDb2shaLU8Bkt+oxLPyAkq8NySoyv3jNyj9x/Zlp4NyafwZ+cO7QfeiPeo2JuPsb0Fmqre2NbZdvzPsOQgmXRHKuQwJKVCchqkpAX/JqdBcmqvf9OC2Gsr4EAddHe++Tipo6OSQ1SS6J04RhcEx5U+hZkIVgLTzKyMIAHcAHywV5mdwKXA/WY2C8gA6kOMSUJS19LGw5GbvzUtbRTnjuL/XTmDD5SX9G/8fvegguqpuJujlqMr7pY+Kviocu2tJ/4sS4L0LEjPCf7NyIas8VAws+/Ku3flnjYmdu34ZsFZcPoYyCke2DE6Dwe/V0/SaD46aRxJFsmpwRl5dMXcezklve/tyWm9Xr32n+zv190Nhxphfw3sr4X9dUf/21oD9Rth2wtBMu9LZj6MGX/ixJGRG7v/vu7ByUx3Z9SrK0jK3Z2Rq7esQf/Y0BKBu3ea2WeApQRdQ+9z9wozuxtY5e6LgS8CPzOzzxOcgn3Y3dX0M0K4Oy9va4jc/K2hs9u5aHoB//DuuVwys/DNY/t0d0HDNqirgLr1wZle/QY4uC+ogE7UpAJB00l69hsVeHoWZE2ILGe/ed+RJo8jy+lZRzd9JKKUdBhTELxGiqQkGJ0fvIrmHL9s5+FIgjiSLGrenDh2vg6ttUHTVm/Jab0SRWGQELs7IxVyr4q6q/PYFXfPemfQ5BW9fqzjHc87vwflHx3473gMNtLq3fLycl+1alWsw0ho+w938rteN38/UF7CTQsnUzZudHBW07oHaiuhLupVvzGqWcKCtvHCWcHZ+FEV+DEq9FNtVhGJ5h5c/fQkidpey1HrXR2QlBJczSSlQFJy5N9jvZKjykaX7+P9yan9P17JQiiYPqCva2ar3b28r336q5J+21zbyq9X7ODxV3axP3Lz97vXTObqombSG16El/7zjco/+vI8a0JQ4Z/7cSicDUWzYdwMSNOgcBJDZkFT36jcAVeu8UKJQI6rI3Lz95EXN7Nv+2vMTq7mB4UNnJNZQ3bLZmxZ1P3/9Oygop/znuDyvXBWsJ6ZF7svICInpEQgR+vugsbtNG9fy/q/rOBA1WvM6NrOL5JqSU6PdFtsToPUGTDlrZHKPlLp50xK7LZ3kRFKiSBRuQc9LXpu3FbidZV0120guauNHGCBG3WpE0iaOAebejOMnxOc4eedprZ6kTiiv+ZE07Ib1j4Ir/wKmnb0bD6YPo71XZN45fAl7EyZzJRZ53LZRRcxecII6lkiIgOiRJAIujphy5Ow+gHYvDR4MrXsIurmfow/1ozlvs2jqG7OZG5xNn91xRS+ctbE4f3kr4gMKiWCeNa4PTjzX/tg0J1zTBFccAcH536QLzzZwp+eqiEtOYl3njmBH75lMvNKhuDJXxEZdpQI4k1nO2z87+Dsf+tzwc3b0y+Dd3wHpl/B3kPdfPT+lazb1cznL5vOzQtLNeqnSIJTIogXezfD6vvhLw8HT+rmlMDFd8H8m4LePMCOfQf4q/tepraljXtvKeey2UWxjVlEhgUlgpGs4xBU/j44+9/5YvAE4oyr4OwPw2lvD55GjHi1uomP/GIl3e48dOtCzi4dG7u4RWRYUSIYiWpeCyr/Vx8NBgzLmwqXfQPmfTAYF6WX5zfV89e/Xk3e6DQe+OgCTou3OQBE5JQoEYwUh1th3eNBAtj9SjAI1uxr4OwPBQ92HeMm7+Orq/ny468yvSiL+z9yLoWa+F1EelEiGM7cYdfqoO1/3RPQcQAKZsGV34Yzrz/u0A3uzo+ff51/+dNGLjg9n5/cfA5ZGRqPXUTeTIlgODrUGDT7rH4gePI3NRPmvjdo+59UfsJhHLq6nbv/UMEDy3dw7byJ/Ov7ziItJWloYheREUeJYLhwhx3/F1T+lb8PxkmfMC8Yf3zu+4LhmPuhraOLLzy6liWv1XDrhWXcddUsknrPCyAiEkWJINb218NfHoJXfgn7tgQjeJ59C5z9VzDhrJM6VPOhDm795Spe3tbAV6+exccvnBpS0CIST5QIYqG7G7Y+E5z9b1wSzEpUshAu/CLMfveAxunf03yID9+3kq179/ODG+dzzVkTBz9uEYlLSgRDbcMS+J8vQ/NOGJUH530yOPsvmDHgQ26qbeVD971Ma1snD3xkAeefPm4QAxaReKdEMJRaa+G3nwwmHX/ffTDzncH8sadg5fYGPnb/SjJSk3n0E29h9sT+3UsQETlCiWAoPfl30HkIPvBLGDftlA/3p3V7uP2RtUwaO4oHPrKAkjxN/SgiJ0+JYKhsewFe/Q1c9KVBSQK/Wr6dry2uYF5JLvd96FzGjk4bhCBFJBEpEQyFznb47y/C2CnBDeFT4O58Z9lG7nn2dS6bVcgPbzxbcweIyClRIhgKy38IezfBTY9B6qgBH6ajq5u7nniNx1ZXc+OCEv7h2rmkJOtBMRE5NaHWImZ2pZltNLMtZnZnH/u/Z2ZrI69NZtYUZjwx0bgdnv9XmPUumLZowIc5cLiTW3+5isdWV3PHZdP4p/ecoSQgIoMitCsCM0sG7gEWAdXASjNb7O6VR8q4++ejyn8WmB9WPDHhHnQVtaRgfKAB2rv/cM9kMt967xncuKB0EIMUkUQX5inlAmCLu29193bgEeDa45S/EXg4xHiG3sYlsOlP8Pa7eiaHOVk79h3gfT9+kU21rdx7S7mSgIgMujDvERQDVVHr1cB5fRU0s8lAGfDMMfbfBtwGUFo6QirC9gPB1UDh7OChsQF4rbqZj9z/Mp3dzoMfX8g5kzWZjIgMvuHSyHwD8Ji7d/W1093vdfdydy8vKCgY4tAG6Pl/huaqYNC45JMf/vn5TfVcf+9y0lOSeeyT5ysJiEhowkwEu4CSqPVJkW19uYF4ahaqrYTl98D8m6F04Um//YlXqvnY/SuZnD+aJz51PqcXakYxEQlPmIlgJTDNzMrMLI2gsl/cu5CZzQTGAstDjGXouAfPDKRnwWV3n+RbnR8/9zpfePQvLCjL4zefWEiRZhQTkZCFdo/A3TvN7DPAUiAZuM/dK8zsbmCVux9JCjcAj7i7hxXLkPrLw8FE8u/6AYzO7/fburqdf/hjJfe/uJ13nTWR77z/TNJT9KCYiITPRlr9W15e7qtWrYp1GH072AD/UQ55p8FHl0JS/y64oieT+fhby/jKOzSZjIgMLjNb7e7lfe3Tk8WD6em74VATvPO7/U4CzYc6uO2Xq3hpWwN/+45Z3HqRJpMRkaGlRDBYqlcFk8wv/BSMP6Nfb6lpbuND973M1r37+f4N87h2XnG4MYqI9EGJYDB0dcIf74Cs8cHDY/2wOTKZTEtbJ/d/ZAEXaDIZEYkRJYLBsPLnUPMavP/+oLfQiYpvb+DjD6wiLSWJ33xiIXMm5oQfo4jIMSgRnKqWPfDMN+G0S4P5hk/g+U313PbLVRTnjuKBj2oyGRGJPSWCU7X0K9DVDu/4V7AT9/T57pObmJg7isf++nzyNJmMiAwDw2WIiZHp9Weg4gm48AuQf9oJi9c0t/GXqibed84kJQERGTaUCAaqoy14gjhvKlxwR7/e8mRlDQCXzy4KMTARkZOjpqGB+r/vQ8NWuPkJSO3fMBDLKmuZOm60xg4SkWFFVwQDse91+N9/gznvgdMv7ddbmg92sPz1fSyaU4T1416CiMhQUSI4We6w5EuQnAZXfKvfb3t2Yx2d3c7ls8eHGJyIyMlTIjhZlb+H15+GS/4Wsif0+23LKmsoyEpnfklueLGJiAzAgBKBmSVmI/fhVvjTXcEQEufe2u+3tXV08dzGehbNLtJgciIy7Az0iqDyxEXi0HPfhtY9cPX3ILn/99n/b8teDrZ3qbeQiAxLx6zNzOwLx9oFJN4VQc06WPFjOOdDUHLuSb11WUUtWekpnH+axhMSkeHneFcE/0Qwc1hWr9eYE7wv/nR3wx8/D6Ny4dKvn9Rbu7qdp9bXcvHMQtJSEutnE5GR4XjtG68Av3P31b13mNnHwwtpGFrzK6h+Ga79EWTmndRbV+9oZN+Bdq6Yo2YhERmejneKugvYYWaf62Nfn7PcxKUD++Cpr0Pp+TDvgyf99mUVNaQlJ/G26QUhBCcicuqOlwhmA2nAR81srJnlHXkBHUMT3jDw1NeC3kJX/1u/BpWL5u4srazh/NPzycpIDSlAEZFTc7ymoZ8CTwNTgdUEN4mP8Mj2+LZzBaz5NZx/OxTNPum3b6hpparhEJ+6+PQQghMRGRzHvCJw9x+4+yzgPnef6u5lUa/4TwJdHcEN4uxJ8LYvD+gQyypqMYNLZxUOcnAiIoPnhJ3h3f2vhyKQYeeln0BdJVz/IKQPrLfs0ooazi4dS2FW/walExGJBfVn7EtzNTz7LZh2Bcy8ekCHqGo4SOWeFvUWEpFhL9REYGZXmtlGM9tiZnceo8wHzKzSzCrM7KEw4+m3P90F3g3v+JeTvkF8xJOVtQAs0iBzIjLMhTYfgZklA/cAi4BqYKWZLXb3yqgy04C7gAvcvdHMYt+YvvlJWL8YLvk7GDtlwIdZWlHD9KIxlI0bPXixiYiEIMwrggXAFnff6u7twCPAtb3K3Arc4+6NAO5eF2I8J9ZxCJb8DeRPg/M/O+DDNBxoZ+X2Bq6Yo6sBERn+wkwExUBV1Hp1ZFu06cB0M/s/M1thZlf2dSAzu83MVpnZqvr6+pDCJZhspnF78MxASvqAD/P0+lq6Hc09ICIjQqxvFqcA04CLgRuBn5lZbu9C7n6vu5e7e3lBQUhP6O7dHEw/ecYHYOrbTulQSytqmZiTwdzi7EEKTkQkPGEmgl1ASdT6pMi2aNXAYnfvcPdtwCaCxDC03IOJ6FNGweXfPKVDHWzv5H8313P5nPGaklJERoQwE8FKYJqZlZlZGnADsLhXmd8RXA1gZuMImoq2hhhT39Y9Dtueh0v/DrJOrbvnC5v2crizW3MPiMiIEVoicPdO4DPAUmA98Ki7V5jZ3WZ2TaTYUmCfmVUCzwJfcvd9YcXUp7ZmWPoVmDAPyj96yodbVlFDzqhUzi07uVFKRURiJbTuowDuvgRY0mvb16KWHfhC5BUbz/wj7K+DGx+BpORTOlRHVzdPb6jj0lmFpCbH+vaLiEj/JHZttXstrPwZnPsxKD77lA+3clsDzYc61FtIREaUxE0E3V3BoHKZ44KHxwbB0ooa0lOSuGi6pqQUkZEj1KahYW31/bD7FXjvz4IpKE+Ru7OsspaLpheQmZa4P6uIjDyJeUWwvw6e/gZMuRDOeP+gHHLdrhb2NLept5CIjDiJmQiW/R20H4SrvzvgQeV6W1pRQ5LBpbOUCERkZEm8RLD9z/DqI3DB7VAwfdAOu6yyhgVleeSNThu0Y4qIDIXESgSd7fDHL0BuKVz4N4N22G17D7Cpdr96C4nIiJRYdzWX/wfs3Qg3/gbSMgftsMsqagBYpPsDIjICJc4VQdNOeP5fYOY7YUafg5wO2LLKWuZMzKYkb/CSi4jIUEmcRLD2oeDG8JXfHtTD1rW28crORjULiciIlThNQ2/7Msy9DnJLTlz2JDxVWYc7XK65iUVkhEqcKwIzGDf4I1wvq6yhNC+TmeOzBv3YIiJDIXESQQha2zp4ccs+Lp9dpLkHRGTEUiI4Bc9trKe9q5vLNTexiIxgSgSnYFllLfmj0zhn8thYhyIiMmBKBAN0uLOLZzfUcdmsIpKT1CwkIiOXEsEALX99H/sPd6q3kIiMeEoEA7SsspbMtGQuOF1zD4jIyKZEMADd3c6TlbVcPKOAjNRTm95SRCTWlAgGYE1VE/Wth7lCvYVEJA4oEQzAssoaUpKMi2cUxjoUEZFTpkRwktydZRW1vOW0fHJGpcY6HBGRUxZqIjCzK81so5ltMbM7+9j/YTOrN7O1kdfHw4xnMGyp28+2vQf0EJmIxI3QBp0zs2TgHmARUA2sNLPF7l7Zq+hv3P0zYcUx2JZV1gKwSFNSikicCPOKYAGwxd23uns78AhwbYifNySWVtRwVkku43MyYh2KiMigCDMRFANVUevVkW29XWdmr5rZY2bW5xjRZnabma0ys1X19fVhxNovu5sO8Wp1M1foITIRiSOxvln8B2CKu58JPAk80Fchd7/X3cvdvbygoGBIA4z21PqgWUiT0IhIPAkzEewCos/wJ0W29XD3fe5+OLL6c+CcEOM5ZUsraphaMJrTC8fEOhQRkUETZiJYCUwzszIzSwNuABZHFzCzCVGr1wDrQ4znlDQf7GDF1gY9RCYicSe0XkPu3mlmnwGWAsnAfe5eYWZ3A6vcfTFwu5ldA3QCDcCHw4rnVD2zsZaubufy2bo/ICLxJdQ5i919CbCk17avRS3fBdwVZgyDZem6Wgqz0jlrUm6sQxERGVSxvlk8IrR1dPH8pnoun1NEkuYeEJE4o0TQD3/evJdDHV3qLSQicUmJoB+WVtSQlZHCwqn5sQ5FRGTQKRGcQGdXN0+tr+WSmYWkpejnEpH4o5rtBFbvaKTxYIeahUQkbikRnMDSilrSUpJ424zYPdEsIhImJYLjcHeWVdbw1tPHMSY91J62IiIxo0RwHOv3tFLdeEgPkYlIXFMiOI6lFTWYwaWae0BE4pgSwXEsq6ylfPJYCrLSYx2KiEholAiOoarhIOv3tKi3kIjEPSWCY1haUQPA5ZqERkTinBLBMSyrrGXm+Cwm54+OdSgiIqFSIujDvv2HWbW9Qb2FRCQhKBH04en1dXQ7XK5JaEQkASgR9GFZZQ3FuaOYMzE71qGIiIROiaCXA4c7eWHzXhbNLsJMcw+ISPxTIujlhU31tHd2q7eQiCQMJYJellXWkpuZyoIpebEORURkSCgRROno6ubp9bVcOrOIlGT9NCKSGFTbRXlpawMtbZ1qFhKRhKJEEGVZZQ0ZqUlcNE1zD4hI4lAiiOjudpZV1HLRtAJGpSXHOhwRkSETaiIwsyvNbKOZbTGzO49T7jozczMrDzOe43ltVzM1LW1coYfIRCTBhJYIzCwZuAe4CpgN3Ghms/solwV8DngprFj6Y1llDclJxiUzC2MZhojIkAvzimABsMXdt7p7O/AIcG0f5f4B+GegLcRYTmhpRS0LpuQxdnRaLMMQERlyYSaCYqAqar06sq2HmZ0NlLj7fx/vQGZ2m5mtMrNV9fX1gx7o6/X72VK3nyvUW0hEElDMbhabWRLwXeCLJyrr7ve6e7m7lxcUDH6PnicrawFYpPsDIpKAwkwEu4CSqPVJkW1HZAFzgefMbDuwEFgcixvGSytqmFucTXHuqKH+aBGRmAszEawEpplZmZmlATcAi4/sdPdmdx/n7lPcfQqwArjG3VeFGNOb1LW0sWZnE1doSkoRSVChJQJ37wQ+AywF1gOPunuFmd1tZteE9bkn68n1QbOQ5h4QkUSVEubB3X0JsKTXtq8do+zFYcZyLEsrapmcn8n0ojGx+HgRkZhL6CeLW9o6WP76Xq6YM15zD4hIwkroRPDcxno6ulxzE4tIQkvoRLC0ooZxY9KYXzo21qGIiMRMwiaCw51dPLehjkWzi0hOUrOQiCSuhE0EL76+jwPtXVyubqMikuASNhEsq6hhdFoybzktP9ahiIjEVEImgq5u58nKWi6eWUhGquYeEJHElpCJYG1VI3v3t6u3kIgICZoIllbUkppsvF1zD4iIJF4icHeWVtTwltPGkZ2RGutwRERiLuESwea6/ezYd1DNQiIiEQmXCJauqwFgkRKBiAiQgIlgWWUt80tzKcrOiHUoIiLDQkIlgt1Nh3htV7MeIhMRiZJQiWBZRdAsdLnmJhYR6ZFYiaCyltMLx3BageYeEBE5ImESQeOBdl7a1qDeQiIivSRMInhmQx1d3a4pKUVEekmYRJA9KpVFs4s4szgn1qGIiAwroc5ZPJwsml2kZwdERPqQMFcEIiLSNyUCEZEEp0QgIpLgQk0EZnalmW00sy1mdmcf+z9pZq+Z2Voz+7OZzQ4zHhERebPQEoGZJQP3AFcBs4Eb+6joH3L3M9x9HvAvwHfDikdERPoW5hXBAmCLu29193bgEeDa6ALu3hK1OhrwEOMREZE+hNl9tBioilqvBs7rXcjMPg18AUgDLunrQGZ2G3AbQGlp6aAHKiKSyGJ+s9jd73H304AvA189Rpl73b3c3csLCgqGNkARkTgX5hXBLqAkan1SZNuxPAL8+EQHXb169V4z2zHAmMYBewf43nik3+No+j3eoN/iaPHwe0w+1o4wE8FKYJqZlREkgBuAD0YXMLNp7r45sno1sJkTcPcBXxKY2Sp3Lx/o++ONfo+j6fd4g36Lo8X77xFaInD3TjP7DLAUSAbuc/cKM7sbWOXui4HPmNllQAfQCHworHhERKRvoY415O5LgCW9tn0tavlzYX6+iIicWMxvFg+xe2MdwDCj3+No+j3eoN/iaHH9e5i7uu6LiCSyRLsiEBGRXpQIREQSXMIkghMNgJcozKzEzJ41s0ozqzAz3bAnGBvLzNaY2R9jHUusmVmumT1mZhvMbL2ZvSXWMcWKmX0+8neyzsweNrOMWMcUhoRIBP0cAC9RdAJfdPfZwELg0wn8W0T7HLA+1kEME98H/uTuM4GzSNDfxcyKgduBcnefS9AN/obYRhWOhEgE9GMAvETh7nvc/ZXIcivBH3lxbKOKLTObRPBA489jHUusmVkOcBHwnwDu3u7uTTENKrZSgFFmlgJkArtjHE8oEiUR9DUAXkJXfgBmNgWYD7wU41Bi7d+B/wd0xziO4aAMqAd+EWkq+7mZjY51ULHg7ruA7wA7gT1As7svi21U4UiURCC9mNkY4HHgjl7DgScUM3snUOfuq2MdyzCRApwN/Njd5wMHgIS8p2ZmYwlaDsqAicBoM7s5tlGFI1ESwckOgBfXzCyVIAk86O5PxDqeGLsAuMbMthM0GV5iZr+ObUgxVQ1Uu/uRq8THCBJDIroM2Obu9e7eATwBnB/jmEKRKImgZwA8M0sjuOGzOMYxxYSZGUH773p3T/gZ4dz9Lnef5O5TCP6/eMbd4/Ksrz/cvQaoMrMZkU2XApUxDCmWdgILzSwz8ndzKXF64zzUsYaGi2MNgBfjsGLlAuAW4DUzWxvZ9pXIuFAiAJ8FHoycNG0FPhLjeGLC3V8ys8eAVwh6260hToea0BATIiIJLlGahkRE5BiUCEREEpwSgYhIglMiEBFJcEoEIiIJTolAZAiZ2cUa4VSGGyUCEZEEp0Qg0gczu9nMXjaztWb208h8BfvN7HuR8emfNrOCSNl5ZrbCzF41s99GxqjBzE43s6fM7C9m9oqZnRY5/Jio8f4fjDy1KhIzSgQivZjZLOB64AJ3nwd0ATcBo4FV7j4HeB74euQtvwS+7O5nAq9FbX8QuMfdzyIYo2ZPZPt84A6CuTGmEjztLRIzCTHEhMhJuhQ4B1gZOVkfBdQRDFP9m0iZXwNPRMbvz3X35yPbHwD+y8yygGJ3/y2Au7cBRI73srtXR9bXAlOAP4f+rUSOQYlA5M0MeMDd7zpqo9nf9So30PFZDkctd6G/Q4kxNQ2JvNnTwPvMrBDAzPLMbDLB38v7ImU+CPzZ3ZuBRjO7MLL9FuD5yOxv1Wb27sgx0s0scyi/hEh/6UxEpBd3rzSzrwLLzCwJ6AA+TTBJy4LIvjqC+wgAHwJ+Eqnoo0frvAX4qZndHTnG+4fwa4j0m0YfFeknM9vv7mNiHYfIYFPTkIhIgtMVgYhIgtMVgYhIglMiEBFJcEoEIiIJTolARCTBKRGIiCS4/w+EGsi1ZxOYBAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(f1s)\n",
    "plt.plot(f1s_eval)\n",
    "plt.title('model f1 micro')\n",
    "plt.ylabel('f1')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'val'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7981764846743296"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max(f1s_eval)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Основной параметр в RNN слое - units. Чем он больше, тем больше весов будет в модели. Он будет соответствовать размеру вектора состояния. \n",
    "\n",
    "Рекуррентные слои можно накладывать друг на друга. В торче проще всего это сделать указав параметр num_layers - он автоматически создаст нужное количество слоев и соединит их между собой. Если num_layers > 1, то в output будут скрытые состояния с последнего слоя, а вот в h - будут последние состояния со всех слоев. Последнее состояние последнего слоя можно выбрать из обоих по индексу.\n",
    "\n",
    "\n",
    "Также можно просто добавить несколько слоев отдельно и передавать в следующий output из предыдущего. Это позволяет создавать слои с разным параметром units (num_layers создает слое и одним и тем же параметром).\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Clf(nn.Module):\n",
    "    \n",
    "    def __init__(self, output_dim):\n",
    "        \n",
    "        super().__init__()          \n",
    "        self.VOCAB_SIZE = len(word2id)\n",
    "        self.EMB_DIM = 50\n",
    "        self.RNN_UNITS = 128\n",
    "        \n",
    "        \n",
    "        self.embedding = nn.Embedding(self.VOCAB_SIZE, self.EMB_DIM)\n",
    "        self.rnn = nn.LSTM(self.EMB_DIM, self.RNN_UNITS, num_layers=3, \n",
    "                           batch_first=True)\n",
    "        self.fc = nn.Linear(self.RNN_UNITS, output_dim)\n",
    "        self.act = nn.LogSoftmax(1) \n",
    "        \n",
    "        \n",
    "    def forward(self, text):\n",
    "\n",
    "        embedded = self.embedding(text)\n",
    "        # применяем rnn слой\n",
    "        rnn_output, (h, *_) = self.rnn(embedded)\n",
    "        # тут тоже будет трехмерный h\n",
    "        # но первый будет не 1 а равный num_layers\n",
    "        # мы возьмем последний с помощью индекса -1\n",
    "        # squeeze(0) тут не нужен и его и не получится применить\n",
    "        output = self.act(self.fc(h[-1]))        \n",
    "        return output\n",
    "\n",
    "\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')  \n",
    "model = Clf(len(label2id))\n",
    "optimizer = optim.Adam(model.parameters(), lr=0.001)\n",
    "criterion = nn.NLLLoss() \n",
    "model = model.to(device)\n",
    "criterion = criterion.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "Loss: 2.3617623233795166; F1 measure: 0.2179296875\n",
      "Loss: 2.334479993581772; F1 measure: 0.21482421875\n",
      "Loss: 2.300838855902354; F1 measure: 0.21990885416666667\n",
      "Loss: 2.2364641678333284; F1 measure: 0.23462890625\n",
      "Loss: 2.1687480256557463; F1 measure: 0.2551015625\n",
      "Eval -  0.3497910680076628\n",
      "1\n",
      "Loss: 1.7937491714954377; F1 measure: 0.384921875\n",
      "Loss: 1.7583761912584306; F1 measure: 0.402421875\n",
      "Loss: 1.7272389276822409; F1 measure: 0.41765625\n",
      "Loss: 1.6929814237356187; F1 measure: 0.43291015625\n",
      "Loss: 1.6524979219436646; F1 measure: 0.4494375\n",
      "Eval -  0.5580244252873563\n",
      "2\n",
      "Loss: 1.3534621369838715; F1 measure: 0.5637890625\n",
      "Loss: 1.347797576189041; F1 measure: 0.5688671875\n",
      "Loss: 1.3199949526786805; F1 measure: 0.5770572916666666\n",
      "Loss: 1.295353894531727; F1 measure: 0.584501953125\n",
      "Loss: 1.2750313370227813; F1 measure: 0.59059375\n",
      "Eval -  0.6372778975095785\n",
      "3\n",
      "Loss: 1.0782435339689256; F1 measure: 0.6580859375\n",
      "Loss: 1.0622646456956863; F1 measure: 0.66775390625\n",
      "Loss: 1.0449156379699707; F1 measure: 0.6735416666666667\n",
      "Loss: 1.0276973754167558; F1 measure: 0.679765625\n",
      "Loss: 1.008345558643341; F1 measure: 0.6875546875\n",
      "Eval -  0.7173114224137931\n",
      "4\n",
      "Loss: 0.8344289755821228; F1 measure: 0.750859375\n",
      "Loss: 0.8304936391115189; F1 measure: 0.750625\n",
      "Loss: 0.8240054986874262; F1 measure: 0.7523567708333333\n",
      "Loss: 0.8129899188876152; F1 measure: 0.7552734375\n",
      "Loss: 0.8034976202249527; F1 measure: 0.7575625\n",
      "Eval -  0.7531734913793104\n",
      "5\n",
      "Loss: 0.6941744810342789; F1 measure: 0.7912109375\n",
      "Loss: 0.6877205783128738; F1 measure: 0.7917578125\n",
      "Loss: 0.6847568049033483; F1 measure: 0.7916796875\n",
      "Loss: 0.6810659089684487; F1 measure: 0.792509765625\n",
      "Loss: 0.6791190255880356; F1 measure: 0.7926796875\n",
      "Eval -  0.7614038553639847\n",
      "6\n",
      "Loss: 0.6025507915019989; F1 measure: 0.81359375\n",
      "Loss: 0.5967927643656731; F1 measure: 0.81748046875\n",
      "Loss: 0.5963641339540482; F1 measure: 0.8163541666666667\n",
      "Loss: 0.5944479934871196; F1 measure: 0.816748046875\n",
      "Loss: 0.5893809888362884; F1 measure: 0.8181171875\n",
      "Eval -  0.7689469588122605\n",
      "7\n",
      "Loss: 0.5028524842858314; F1 measure: 0.84296875\n",
      "Loss: 0.5039751522243023; F1 measure: 0.84279296875\n",
      "Loss: 0.5085151822368303; F1 measure: 0.8419401041666666\n",
      "Loss: 0.5065407755225897; F1 measure: 0.84267578125\n",
      "Loss: 0.5092811302542687; F1 measure: 0.842171875\n",
      "Eval -  0.7737170737547893\n",
      "8\n",
      "Loss: 0.4210300129652023; F1 measure: 0.87296875\n",
      "Loss: 0.4256823307275772; F1 measure: 0.869765625\n",
      "Loss: 0.4290482051173846; F1 measure: 0.8676953125\n",
      "Loss: 0.43412051871418955; F1 measure: 0.866142578125\n",
      "Loss: 0.4374466893672943; F1 measure: 0.864578125\n",
      "Eval -  0.7783758381226054\n",
      "9\n",
      "Loss: 0.3661569781601429; F1 measure: 0.8903125\n",
      "Loss: 0.37188968479633333; F1 measure: 0.8887109375\n",
      "Loss: 0.3699196836352348; F1 measure: 0.887890625\n",
      "Loss: 0.3715294837579131; F1 measure: 0.886689453125\n",
      "Loss: 0.3757590047419071; F1 measure: 0.884796875\n",
      "Eval -  0.7666109913793103\n"
     ]
    }
   ],
   "source": [
    "f1s = []\n",
    "f1s_eval = []\n",
    "\n",
    "for i in range(10):\n",
    "    print(i)\n",
    "    f1s.append(train(model, training_generator, optimizer, criterion, 100))\n",
    "    evl = evaluate(model, valid_generator, criterion)\n",
    "    print('Eval - ', evl)\n",
    "    f1s_eval.append(evl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAziklEQVR4nO3dd3iUZdbH8e9JL4SEhBaSQEKTXiQEFLuiFBXXiqu4VnTVtezuu4u+vuq6rmVXXXVF1+5aUVEU3aiAAiooVZAOoSahBdII6cl5/5gBh5BAAvNkksz5XNdcmXnanBnl/s3T7ltUFWOMMf4rwNcFGGOM8S0LAmOM8XMWBMYY4+csCIwxxs9ZEBhjjJ+zIDDGGD9nQWD8koi8ISIP13PZLSJyTh3zwkXkMxEpEJEPvVvlYe/VWUSKRCTQyfcx/seCwJjjcynQAYhT1ctEJF5EpovIdhFREUn21hup6jZVbaWqVd7apjFgQWDM8eoCrFfVSvfrauBL4BLflXQoEQnydQ2mabMgME2W+5DM/4jIzyKyX0ReFZEOIvKFiOwTkVki0sZj+QtFZJWI5IvIHBHp7TFvsIgsda/3PhBW473OF5Fl7nXni8iAetT3F+B+4Ar3IZsbVHWXqj4PLPL2ZxSRZPdeRpD7dayIvO7e+8gTkU/c088QkSwR+bOI7AReF5FQEXnavex29/PQ+tRoWj4LAtPUXQKMBHoCFwBfAPcC7XD9/3sHgIj0BN4D7nLPSwc+E5EQEQkBPgHeAmKBD/H4xS4ig4HXgJuBOOBFYPrRGkpVfQB4BHjffcjmVSc/Yy3eAiKAvkB74J8e8zri+qxdgInA/wLDgUHAQCANuO8Y6zUtjAWBaer+5f6VnQ18ByxQ1Z9UtRSYBgx2L3cF8F9VnamqFcATQDhwMq4GMBh4WlUrVHUqh/5inwi8qKoLVLVKVf8DlLnXawz1/YwHiUg8MBq4RVXz3J9rrsci1cADqlqmqiXAVcBDqrpbVXOAvwATnP5gpnmwY4emqdvl8bykltet3M87AVsPzFDVahHJBBKAKiBbD+1hcavH8y7Ab0Tkdx7TQtzbbAz1/YyekoBcVc2rY5s57iA54JDvx/28sT6faeJsj8C0FNtxNegAiIjgaiyzgR1AgnvaAZ09nmcCf1PVGI9HhKq+1xiFH6NMIFZEYuqYX7Nb4UO+H1yff7sDdZlmyILAtBQfAGNF5GwRCQb+gOvwznzgB6ASuENEgkXkYlzHyA94GbhFRIaJS6SIjBWRqGMpRETCgAPnF0Ldr71KVXfgOpfwvIi0cX+u046wynvAfSLSTkTa4jrJ/ba36zLNkwWBaRFUdR1wNfAvYA+uk64XqGq5qpYDFwPXArm4zid87LHuYuAm4DkgD8hwL3usSoAi9/O17tdOmABUuN9jN64T5XV5GFgM/AysAJa6pxmD2MA0xhjj32yPwBhj/JwFgTHG+DkLAmOM8XMWBMYY4+ea3Q1lbdu21eTkZF+XYYwxzcqSJUv2qGq72uY1uyBITk5m8eLFvi7DGGOaFRHZWtc8Rw8NicgoEVknIhkiMqmW+V1E5Gt3z4tzRCTRyXqMMcYczrEgcI+iNBlXx1h9gCtFpE+NxZ4A3lTVAcBDwKNO1WOMMaZ2Tu4RpAEZqrrJfWfnFGBcjWX6AN+4n8+uZb4xxhiHOXmOIAFXx1gHZAHDaiyzHNet/88AvwKiRCROVfd6LiQiE3F1FUznzp2pqaKigqysLEpLSw+b15KEhYWRmJhIcHCwr0sxxrQgvj5Z/EfgORG5FvgWV0+Rh43HqqovAS8BpKamHtYnRlZWFlFRUSQnJ3NoB5Mth6qyd+9esrKySElJ8XU5xpgWxMkgyMbVDfABie5pB6nqdlx7BIhIK+ASVc1v6BuVlpa26BAAEBHi4uLIycnxdSnGmBbGyXMEi4AeIpLiHipwPDDdcwERaSsiB2q4B9dwgcekJYfAAf7wGY0xjc+xIFDVSuB24CtgDfCBqq4SkYdE5EL3YmcA60RkPdAB+JtT9RhjTHOjquwoKGHW6l08M2sDK7MLHHkfR88RqGo6rkHEPafd7/F8KjDVyRoaQ35+Pu+++y633nprg9YbM2YM7777LjExMc4UZoxpNqqrlS1797Nqe6H7UcDq7YXs3V8OgAjEtgqhX0K019/b1yeLW4T8/Hyef/75w4KgsrKSoKC6v+L09PQ65xljWq6Kqmo27CpipbuxP9Do7y93XSsTHCj0aB/F2b3b07dTNH07taZ3fGsiQ51psi0IvGDSpEls3LiRQYMGERwcTFhYGG3atGHt2rWsX7+eiy66iMzMTEpLS7nzzjuZOHEi8Et3GUVFRYwePZpTTjmF+fPnk5CQwKeffkp4eLiPP5kx5ngVl1eyZsc+Vm8vYGV2Iat2FLB+ZxHlVdUARIQE0ju+NZcMSaRfp2j6dGpNzw5RhAQ1Xp+gLS4I/vLZKlZvL/TqNvt0as0DF/Stc/5jjz3GypUrWbZsGXPmzGHs2LGsXLny4GWer732GrGxsZSUlDB06FAuueQS4uLiDtnGhg0beO+993j55Ze5/PLL+eijj7j66qu9+jmMMc7KLy4/eFhnZbbr7+Y9+6l2X/TeJiKYvp2iuW5EMn06taZfQjTJcZEEBvj2QpAWFwRNQVpa2iHX+j/77LNMmzYNgMzMTDZs2HBYEKSkpDBo0CAAhgwZwpYtWxqrXGNMA6kqOwtLWZXtOp5/4BBPdv4vw1N3ig6jT6dozh/Qib6dWtM3IZpO0WFN8uq/FhcER/rl3lgiIyMPPp8zZw6zZs3ihx9+ICIigjPOOKPWO6BDQ0MPPg8MDKSkxKnxzo0xDVHbSdxV2wvJ9TiJmxIXyYld2jDhpC6uRr9TNLGRIT6uvP5aXBD4QlRUFPv27at1XkFBAW3atCEiIoK1a9fy448/NnJ1xpj6cl2uWcryzHyWZxWwPDOfFdkFFJVVAr+cxD2nkU7iNpbmXX0TERcXx4gRI+jXrx/h4eF06NDh4LxRo0bx73//m969e3PCCScwfPhwH1ZqjPGUX1x+sMH/OSufZZkF7CkqA1yNfu/41lw0uBMDEmJ8chK3sYjqYV33NGmpqalac2CaNWvW0Lt3bx9V1Lj86bMa400l5VWs2l7AMvev/Z+z8tm6t/jg/G7tIhmYFMPAxBgGJsXQOz6K0KBAH1bsXSKyRFVTa5tnewTGmBansqqadbv28bP71/7yrALW79pHlfvynU7RYQxIjGH80M4MTIymX2I0rcP8t1dfCwJjTLOmqmzdW8zyrHyWZxawPCufVdsLKK1wXacfHR7MgMRozundjQGJMQxMjKZ96zAfV920WBAYY5qV3YWlBw/tLMvM5+esAgpKKgAICw6gX6dofp3WhYFJ0QxMjKFLXESTvGSzKbEgMMY0WYWlFazMKmBZVj4/u3/t7yhwXX4dGCD07BDF6H4dDx7b79mhFUGBLe9krtMsCIwxTUJFVTVrd+xjWVY+y7blsywzj405+w/O7xIXwdDkWAYkRjMoKYa+naIJD2k5J3N9yYLAGNPoVJWsvBJ+ynQ1+suz8lmZXUBZpeu4flxkCIOSYhg3KIGBSTEMSIimTTO6Qau5sSDwgVatWlFUVOTrMoxpNAXFFSzLymd5puu4/vLM/IPdK4cGBdA/IZoJw7swMCmGQUkxJLYJt+P6jciCwBjjVeWV1azZUcgyj0Z/0x7XIR4R6N6uFWf2as8gd6N/Qscogu24vk9ZEHjBpEmTSEpK4rbbbgPgwQcfJCgoiNmzZ5OXl0dFRQUPP/ww48aN83GlxnjXgUs3DzT6yzLzWb298GAXy+2iQhmUFMMlQxIZlBRDfz+/Xr+panlB8MUk2LnCu9vs2B9GP1bn7CuuuIK77rrrYBB88MEHfPXVV9xxxx20bt2aPXv2MHz4cC688ELb3TXNWt7+8oMnc5e7D/XkFbsu3QwPDqR/YjTXjkg++Gs/von2tmkO1fKCwAcGDx7M7t272b59Ozk5ObRp04aOHTty99138+233xIQEEB2dja7du2iY8eOvi7XmHoprahi9Y7Cg8f1l2X+0iWDCPRsH8W5fToyqLNdutnctbwgOMIvdydddtllTJ06lZ07d3LFFVfwzjvvkJOTw5IlSwgODiY5ObnW7qeNaSr2FpWxaEseCzfnsmRrLqt3FFJR5eqSoWPrMAYlubpkOHCIp1Uz73HT/ML+S3rJFVdcwU033cSePXuYO3cuH3zwAe3btyc4OJjZs2ezdetWX5dozCF2FJSwcHPuwceG3a4r2cKCAxiUFMMNp3Q9eIinY7R1ydCSWRB4Sd++fdm3bx8JCQnEx8dz1VVXccEFF9C/f39SU1Pp1auXr0s0fkxV2ZZbzAKPhn9bruswT1RoEKnJbbj4xETSUmLpnxDdIrtaNnWzIPCiFSt+OUndtm1bfvjhh1qXs3sIjNOqq5WMnCKPhn8vuwpd/ezHRoaQlhzLtScnk5YSS+/41j4fM9f4lqNBICKjgGeAQOAVVX2sxvzOwH+AGPcyk1Q13cmajGmJKquqWbNjHws272Xh5lwWbck9eDVPx9ZhDO8aR1pKLMNSYunWrpVdyWMO4VgQiEggMBkYCWQBi0Rkuqqu9ljsPuADVX1BRPoA6UCyUzUZ01KUVVaxIqvg4C/+JVvzDg6nmBwXwcg+HUhLiWNYSqzdpWuOysk9gjQgQ1U3AYjIFGAc4BkECrR2P48Gth/rm6lqi/+fvbmNJme8p7i8kqVb81m4xXWY56dt+Qf75TmhQxS/GpxAWkosaSmxdLC+9k0DORkECUCmx+ssYFiNZR4EZojI74BI4JzaNiQiE4GJAJ07dz5sflhYGHv37iUuLq7FhoGqsnfvXsLC7B+5PygoqWDxFtev/QWbc1mZXUBltRIg0M/dL09aSixDk2OtMzZz3Hx9svhK4A1VfVJETgLeEpF+qlrtuZCqvgS8BK4xi2tuJDExkaysLHJychqlaF8JCwsjMTHR12UYB1RWVfPjplxmrdnFgs25rN1ZiCqEBAYwMCmam0/vSlpKHCd2jiHKumgwXuZkEGQDSR6vE93TPN0AjAJQ1R9EJAxoC+xuyBsFBweTkpJyHKUa0/gqqqqZv3Ev6T/vYMbqneQVVxAeHMiQLm24+5yepKXEMigphrBg63PfOMvJIFgE9BCRFFwBMB74dY1ltgFnA2+ISG8gDGjZP+uNXyuvrGZexh7SV+xgxupdFJRU0Co0iHN6t2d0/3hO79nOGn7T6BwLAlWtFJHbga9wXRr6mqquEpGHgMWqOh34A/CyiNyN68TxtWpnRE0LU1ZZxXfr95C+cgczV+9iX2klUWFBjOzTgTH94jmlR1tr/I1POXqOwH1PQHqNafd7PF8NjHCyBmN8obSiirnrc/hixQ5mrdlNUVkl0eHBjOrbkTH94zm5exyhQdb4m6bB1yeLjWkxSsqrmLt+N/9dsZNv1uxif3kVMRHBjO0fz5gB8ZzUNc66bmhsVRVQXgTlxVBRDOX7XY8Dz2ubdtj8YghtBa07QetE999OEO1+Hhrl60953CwIjDkOxeWVzF6bQ/qKHXyzdjclFVXERoZw4aAExvTvyPCucTb61tFUV0PFfleDW150aIN8xEbbc/lij23s/+V5dUXDagmOgJDIQ/8Gh8P+PbBjOeyv5RRmaLRHOCRA6wOP5hMWFgTGNFBRWSXfrN3NFyt2MHvdbkorqmnbKpRLhiQwpl88aSmxLbNf/qrKIzTURTUa4aM17B7LV5Y0rI6gMHcjHQkhEb802lHx7ucR7nmRHs/rmOa5naBwCDjKf7fKMti3AwqyoXA7FGa5/26HgizXoFj7a7noMbS1Rzh4BMWB0IhO8GlYWBAYUw/7Siv4es1u0lfsYO76HMoqq2kXFcrlqUmM6R/P0OTYpt9xW2U5lOZDSd4vj+LcQ1+X5EFJLpQWHN6wV5U34M3E3ege+HXdytXYhrWG1vGHNsohrX5pzGtbp+bzAB+eWwkKhTbJrkddDoRF4XZ3YBx4bHf9PWJYdDp8b8JzDyOs9eHreeNjObJVY1qAgpIKvl6zi/QVO/h2/R7Kq6rp2DqMK9M6M3ZAPEM6tyHAF41/ZXntjXfNaQcb+XzX3/J9dW9TAiC8jccjFmI6H73BPviLusYyweGuYcz8Ub3Cohz2bT90b+JAUBwpLEb/A4ZN9H7JXt+iMc1YfnE5M1e7Gv/vM/ZQUaV0ig5jwkldGNO/I4OTvNj4V1e7fqEX50Lx3l8edTbw+e4G/QjdmEvgoQ16VDx06Fujka/lEdr66IdFjPcEhdQzLHYcujfRebgz5TiyVWOakYqqaj75KZvPf97BvIw9VFYrCTHhXDcihdH9OjIwMebojb+q63BK8d5aGvbcOqbnwaG9qfwiIOjQhrp1InTo7zEtBiJia2/Q/fWXeEsTFAJturgeTr+V4+9gTBO2PDOfSR+vYM2OQjrHRnDDqSmM7deR/u0CkOJcKN4EGZ6Nuecj99DnWlX7mwQEQ0Sc+xEL7fu4/h6cFvfL6/BY1/OQVtagm0ZjQWD80v6ySp6csZ70+Uu5NGIpb3ZZR1vNRVblwqK9dV92KIGHNt5te0DE8BqNukeDHhHnuhrEGnXThFkQGL8zb9lqfvj8dc4r/5b7QtcRUKVQ3QviukNiao3GvMYv9rBoa9RNi2NBYPzD/r3sW/YxO+a9y/D9PzFClJI23QkYPAn6/graneDrCo3xGQsC03IV58Laz9FV09BNc4nSKnI0nkVJ13PimOsI79Tf1xUa0yRYEJiWpbQA1v4XVk2Djd9AdSW7A+OZWjGWje3P47eXX8jwjs7clGNMc2VBYJq/0kJY/yWs/Bg2fg1V5Wh0EssTfs1ft/RinXbjz2N78cSwLr65AcyYJs6CwDRPZUWuxn/VNNgwE6rKXLfhp00ko91I7vwugFUb9jGyTwdmjutLfHS4rys2psmyIDDNR3kxbJgBqz6G9TNcnZW16gip10HfX1Hc4UT+OSuDVz/cTFyrUF646kRG9euI2FU+xhyRBYFp2ipKIWOWq/Ff96WrA7TIdjD4Kuh7seuW+4BAvl2fw71Pf09WXglXpnVm0uheRIfbIO/G1IcFgWl6KstcJ3pXTYO16a7O0sJjYcBlrsY/+ZSDPVDm7i/nr5+vYNpP2XRtG8n7E4czrGucjz+AMc2LBYFpGqoqYNMcV+O/5nMoK4CwGOh7kes6/5TTIPCXX/iqyrSfsvnr56vZV1rJ787qzm1ndrexf405BhYExre2zodl78Laz12dsIVGQ6+x0O9iSDnd1fFWDZm5xdw7bQXfbdjD4M4xPHbxAE7o2LRHgDKmKbMgML5RWQ4z74cFL0BIFPQa4/rl3+0sV3/uta1SVc3r87bw1Mz1BAj85cK+XD28S9MfEMaYJs6CwDS+vK0w9TrIXgLDboFzHnQNZHIEK7MLuOfjFazILuCc3u15aFw/OsXYJaHGeIMFgWlca9Phk1tc/fdf/ib0GXfExUvKq3h61npe+X4zbSJCmPzrExnT3y4JNcabHA0CERkFPAMEAq+o6mM15v8TONP9MgJor6oxTtZkfKSqAmY9CD88B/ED4bI3ILbrEVf5fsMe7p22gm25xVyRmsS9Y3oTHWGXhBrjbY4FgYgEApOBkUAWsEhEpqvq6gPLqOrdHsv/DhjsVD3Ghwqy4MPrIGshDL0Rzv0bBIfVuXje/nIe/u8aPlqaRUrbSN67aTgndbNLQo1xipN7BGlAhqpuAhCRKcA4YHUdy18JPOBgPcYXNsyEjydCVTlc+hr0u6TORVWV6cu389BnqykoqeC2M7vxu7N62CWhxjjMySBIADI9XmcBw2pbUES6ACnAN3XMnwhMBOjcubN3qzTOqKqE2Q/D9/+EDv3gsv9A2+51Lp6ZW8x9n6xk7vocBibF8PbF/ekdb72EGtMYmsrJ4vHAVNXaB31V1ZeAlwBSU1O1MQszx6BwO0y9AbbNhxN/A6Mfr/OqoKpq5fV5m3lyxnpE4IEL+nDNScl2SagxjcjJIMgGkjxeJ7qn1WY8cJuDtZjGkvG161BQRTH86iUYeEWdi+bsK2PiW4v5aVs+Z57Qjr9e1I/ENhGNWKwxBpwNgkVADxFJwRUA44Ff11xIRHoBbYAfHKzFOK26CuY8Bt/+A9r1gsv/c8ThHzNzi5nw6gJ2FpbyzPhBXDiwk10SaoyPOBYEqlopIrcDX+G6fPQ1VV0lIg8Bi1V1unvR8cAUVbVDPs3Vvl3w0Q2w5TsYdDWM+QeE1P3Lft3OfUx4dQFlldW8c+NwhnRp04jFGmNqcvQcgaqmA+k1pt1f4/WDTtZgHLZpLnx0I5Ttg3HPu7qHPoIlW3O57vVFhIcE8uEtJ9Gzg/URZIyvNZWTxaa5qa6C756EOY9CXHe45lPo0OeIq8xeu5vfvrOE+Ohw3rw+jaRYOx9gTFNgQWAarigHPr4JNs2G/pfD+f+E0FZHXOWTn7L544fL6RUfxRvXpdG2Ve0dyxljGp8FgWmYLfNc5wOKc+GCZ+HEa+AoJ3lf+34zD32+mpO7xfHihCFEhVk3EcY0JRYEpn6qq2HeP+Gbv0GbZLjpQ+jY/4irqCpPzljPc7MzGNW3I0+PH2R3CRvTBFkQmKPbvxem3QwZM11DRV7wDIQd+a7fqmrl/z5dybsLtnFlWhIPX9TfbhIzpomyIDBHtm2Ba+yA/Tkw9klIveGoh4LKKqu4+/1lpK/YyW1nduOP555g9wgY04RZEJjaqcL8f8HXf4HoRLhhJnQadNTVisoqmfjmYuZv3Mt9Y3tz46lH7mraGON7FgTmcCV58MmtsC4del8A4yZDWPRRV9tbVMa1ry9i9Y5CnrxsIJcMSWyEYo0xx8uCwBwqawl8eC3s2wGjHodhNx/1UBBAdn4JE15ZQHZ+CS9NGMLZvTs4X6sxxissCIyLKiz4N8z4P4iKh+u/gsQh9Vp1w659THh1IcXllbx94zCGJsc6XKwxxpssCAyU5MP022HNZ3DCGLjoeQivX/8/S7flcf0biwgJDOD9m0+yMQSMaYYsCPzd9mXw4W9cw0me+zCcdHu9DgUBzF2fwy1vLaF961DevmGYdRlhTDNlQeCvVGHRK/DVvRDZHq77ApLS6r369OXb+cMHy+jRPor/XJ9GuyjrMsKY5sqCwB+VFsJnd8CqadDjXPjVixBR/+P6b/6whQemryItOZaXf5NKa+sywphmzYLA32xf5roqKH8bnP0AjLgLAgLqtaqq8vSsDTzz9QZG9unAv64cbF1GGNMCWBD4i0MOBbWD69Kh8/B6r15VrTw4fRVv/biVy4Yk8ujF/QkKrF+AGGOaNgsCf1BaANN/B6s/hR7nwUUvQGRcvVcvr6zm9x8s4/Ofd3DzaV2ZNLqXdRlhTAtiQdDSZS91HQoqzIaRf3VdFVTPQ0EA+8squeXtJXy3YQ/3jO7Fzad3c65WY4xPWBC0VKqw4EWYcR+06tDgq4IAcveXc90bi1iZXcDfLx3A5alJDhVrjPElC4KWqCQPPr0d1n4OPUe7bhBrwFVBANvzS5jw6gIy80r499VDGNnHuowwpqWyIGhpspbA1GuhcDuc9wgMv7XeN4gdkLG7iGteXcC+0kreuj6NYV3rfz7BGNP8WBC0FKrw4/Mw8wGPvoJSG7yZ5Zn5XPv6QgIDAphy83D6djp6r6PGmObNgqAlKM6FT29zdRvd63wY91y9+wry9P2GPUx8azFxrUJ4+4ZhdImLdKBYY0xT4+iF4CIySkTWiUiGiEyqY5nLRWS1iKwSkXedrKdFylwEL54GG2a6uo2+4u1jCoH//ryD695YSOfYCD665WQLAWP8iGN7BCISCEwGRgJZwCIRma6qqz2W6QHcA4xQ1TwRae9UPS1OdTX88JxrBLHWCXDDV5BQv26ja3r7x63836crSe3Shld+M5TocOsywhh/4uShoTQgQ1U3AYjIFGAcsNpjmZuAyaqaB6Cqux2sp+UozoVpt8CGr6D3hXDhvyA8psGbUVX+9U0GT81cz9m92vPcr08kPMS6jDDG3zgZBAlApsfrLGBYjWV6AojIPCAQeFBVv6y5IRGZCEwE6Ny5syPFNhueg8mPeQKG3tjgq4IAqquVhz5fzRvzt3DxiQk8fskAgq3LCGP8kq9PFgcBPYAzgETgWxHpr6r5ngup6kvASwCpqanayDU2DdXVMP8Z+PqvENMZbpgBnQYf06ZUlT9/9DMfLsnixlNSuHdMbwICrMsIY/yVk0GQDXjeipronuYpC1igqhXAZhFZjysYFjlYV/Ozf4/rUFDGTOj7K7jgmXoNJl+X1+dt4cMlWdxxVnfuHtnT+g0yxs85eSxgEdBDRFJEJAQYD0yvscwnuPYGEJG2uA4VbXKwpuZn63z496mw+VsY+xRc+vpxhcDCzbk8kr6Gc/t0sBAwxgAO7hGoaqWI3A58hev4/2uqukpEHgIWq+p097xzRWQ1UAX8j6rudaqmZqW6Gr5/CmY/Am2S4cZZED/guDa5u7CU295dSufYCJ64fKCFgDEGcPgcgaqmA+k1pt3v8VyB37sf5oCiHJg2ETZ+A/0ugfOfhrDjGxS+oqqaW99ZSlFpJe/cOMxGFTPGHHRMQSAirVS1yNvFGGDL9zD1BlfHcec/DUOuPaargmp6JH0Ni7fm8eyVg+nZIeq4t2eMaTmOdY9gNeDn13F6WXUVfPckzHkUYrvC1R9Bx35e2fSny7J5fd4Wrh+RwoUDO3llm8aYlqPOIBCRug7XCNDKmXL8VNFu+Pgm2DQH+l8O5z8Fod751b52ZyGTPlpBWnIs94zp5ZVtGmNaliPtETwC/AOorGWe3XnkLZvmukKgtMB1h/DgCV45FARQWFrBLW8tISosiOeuGmw3jBljanWkIFgKfKKqS2rOEJEbnSvJT1RXwdy/w9zHoW0PmPAJdOjjvc1XK79/fzlZeSVMmTic9lFhXtu2MaZlOdJPxGxgq4jcWcu8hnd0b36xbxe8OQ7mPgYDx8NNs70aAgAvzN3IrDW7uG9sb1KTGzY6mTHGvxxpj6APEAJcLyJv4jo3cECFo1W1ZNsWwPtXQVkRjHseBl/l9bf4dn0OT8xYx7hBnfjNycle374xpmU5UhC8CHwNdAWWcGgQqHu6aYiqCph2M4REwm8+g/a9vf4WWXnF3DnlJ07oEMWjF/e3m8aMMUdVZxCo6rPAsyLygqr+thFrarmWT4G8zXDlFEdCoLSiit++vZTKauXfVw8hIsTXfQoaY5qDo7YUFgJeUlkO3/7d1WNoz1GOvMWD01exIruAl69JJbmtjTBmjKkfu56wsSx7G/K3wZn/67XLQz1NWbiNKYsyuf3M7ozs08Hr2zfGtFwWBI2hsgy+fQIS06D7OV7f/PLMfO7/dBWn9mjL3SN7en37xpiWzQ4iN4Yl/4HCbBg32et7A7n7y7n1naW0iwrl2fGDCbQBZowxDWRB4LSKElcfQl1GQNczvLrpqmrljvd+IqeojI9uOZk2kSFe3b4xxj9YEDht8etQtBMufdXrewNPzVzH9xl7+PslA+ifeOyD1Rhj/JudI3BS+X7X4DIpp0HyKV7d9IxVO5k8eyNXpiVx+dCko69gjDF1sD0CJy16BfbnwJlve3Wzm3KK+MMHyxmQGM0DF/T16raNMf7H9gicUrYPvn8aup0NnYd7bbPF5ZXc8vYSggKFF64eQlhwoNe2bYzxT7ZH4JQFL0JJruu+AS9RVf780Qoydhfx5vXDSIgJ99q2jTH+y/YInFBaAPP/5bqDOHGI1zb7+rwtfLZ8O3849wRO6dHWa9s1xvg3CwIn/PgClObDGfd4bZMLN+fySPoazu3TgVvP6Oa17RpjjAWBt5XkwQ+Todf50GmQVza5u7CU295dSlJsBE9cPtB6FDXGeJWdI/C2HyZDWaHX9gYqqqq59Z2lFJVW8vYNw2gdFuyV7RpjzAGO7hGIyCgRWSciGSIyqZb514pIjogscz+a9xCY+/e6Dgv1/RV07OeVTT6SvobFW/N4/NIBnNDROwPaG2OMJ8f2CEQkEJgMjASygEUiMl1VV9dY9H1Vvd2pOhrV/GddN5GdfljmHZNPl2Xz+rwtXD8ihQsHdvLKNo0xpiYn9wjSgAxV3aSq5cAUYJyD7+dbRbth4UvQ/1Jo3+u4N7d2ZyGTPlpBWnIs94w5/u0ZY0xdnAyCBCDT43WWe1pNl4jIzyIyVURq7StBRCaKyGIRWZyTk+NErcdv3jNQWeqVvYHC0gpueWsJUWFBPHfVYIID7Zy+McY5vm5hPgOSVXUAMBP4T20LqepLqpqqqqnt2rVr1ALrZd9OV3cSA8ZD2+7HtanqauX37y8nK6+EyVedSPuoMC8VaYwxtXMyCLIBz1/4ie5pB6nqXlUtc798BfDe3VeN6bunXAPTn/4/x72pF+ZuZNaaXfzv2N4MTY71QnHGGHNkTgbBIqCHiKSISAgwHpjuuYCIxHu8vBBY42A9zijIhiWvw+CrILbrcW3q2/U5PDFjHeMGdeLak5O9U58xxhyFY1cNqWqliNwOfAUEAq+p6ioReQhYrKrTgTtE5EKgEsgFrnWqHsd89wSowmnHtzeQlVfMnVN+omf7KB69uL/dNGaMaTSO3lCmqulAeo1p93s8vwfwXj8MjS1vKyx9C068BmI6H/NmSiuq+O3bS6msUv49YQgRIXafnzGm8ViLczy+e8I16tipfziuzTw4fRUrsgt4+ZpUUtpGeqk4Y4ypH19fNdR85W6Cn96BIddBdG1XxdbPlIXbmLIok9vP7M7IPh28WKAxxtSPBcGxmvsPCAyGU39/zJtYnpnP/Z+u4tQebbl7ZE8vFmeMMfVnQXAs9mTAz1Ng6I0Q1fGYNpG7v5xb31lKu6hQnh0/mMAAOzlsjPENO0dwLOY+BkFhMOKuY1q9qlq5472fyCkq46NbTqZNZIh36zPGmAawPYKG2r0WVkyFtInQ6tjucn5q5jq+z9jDw+P60T8x2ssFGmNMw1gQNNScRyEkEk6+45hWn7V6F5Nnb+TKtCQuH1pr10rGGNOoLAgaYudKWP0JDP8tRMY1ePXSiiru/3QlveNb88AFfb1fnzHGHAMLgoaY8yiERsNJtx3T6q/P28L2glLuP78PYcGBXi7OGGOOjQVBfW1fBms/d4VAeJsGr567v5znZ2dwdq/2nNSt4XsTxhjjFAuC+przKITFwPBbjmn1577JYH95JZNG2yAzxpimxYKgPrIWw/ov4eTfQVjDr/LZtreYt37cwhVDk+jRwcYdNsY0LRYE9TH7EYiIg2E3H9Pqf/9qLUEBAdx1jt09bIxpeiwIjmbbj7DxaxhxJ4Q2/Nf8ssx8Pv95BzedmkKH1jbamDGm6bEgOJrZf4PI9jD0pgavqqo8mr6Gtq1CmHh6NweKM8aY42dBcCSbv4PN38Ipd0NIRINX/2btbhZszuXOc3rSKtR68zDGNE0WBHVRdV0pFBUPqdc1ePXKqmoe/WItXdtGMt7uIDbGNGEWBHXZNAe2znMNOhMc3uDVP1ySRcbuIv40qhfBgfY1G2OaLmuhaqPqulKodYJrGMoGKi6v5KmZ60nt0obz+tpgM8aYps2CoDYZsyBrIZz2RwgKbfDqr3y3mZx9ZdwzprcNQm+MafIsCGpSdV0pFNMZBl3d4NVz9pXx4tyNjO7XkSFdGt4VhTHGNDYLgprWfwnbf4LT/gRBDR8w5pmv11NWWc3/nHeCA8UZY4z3ORoEIjJKRNaJSIaITDrCcpeIiIpIqpP1HFV1tWtvoE0KDLyywatvzCnivYWZ/HpYZ7q2a+VAgcYY432OBYGIBAKTgdFAH+BKEelTy3JRwJ3AAqdqqbe1n8POFXDGJAhs+HX/j3+xlvDgQO44u4cDxRljjDOc3CNIAzJUdZOqlgNTgHG1LPdX4HGg1MFajq662nXfQFwP6H9Zg1dftCWXGat3ccvpXWnbquEnmI0xxlecDIIEINPjdZZ72kEiciKQpKr/dbCO+lk9DXavdu0NBDRs0BhV5ZH0NXRoHcoNp3R1qEBjjHGGz04Wi0gA8BTwh3osO1FEFovI4pycHO8XU10Fcx6Ddr2h78UNXv3LlTv5aVs+vx/Zk/AQG3nMGNO8OBkE2YBn3wqJ7mkHRAH9gDkisgUYDkyv7YSxqr6kqqmqmtquXTvvV7piKuxZD2feAwEN+0rKK6t5/Mu19OzQikuHWFcSxpjmx8kgWAT0EJEUEQkBxgPTD8xU1QJVbauqyaqaDPwIXKiqix2s6XBVlTD3MejQH3pd0ODV31u4jS17i7lndG8CA+zmMWNM8+NYEKhqJXA78BWwBvhAVVeJyEMicqFT79tgP78PuZuOaW9gX2kFz3y9gZO6xnHGCQ7sqRhjTCNwtG9kVU0H0mtMu7+OZc9wspZaVVXA3MchfhCcMKbBq784dxO5+8u517qSMMY0Y/59Z/GydyB/K5z5v9DAhnxnQSmvfL+JcYM60T+x4eMYG2NMU+G/QVBZBnP/AYlDocfIBq/+1Mx1VFfDH8+1riSMMc2b/wbB0jehMAvOvLfBewNrdxYydUkW15zUhaTYho9cZowxTYl/BkFFKXz3JHQ+Cbqe2eDVH/9iLa1Cg7j9rO4OFGeMMY3LP4Ngyeuwb8cxnRuYn7GH2etyuP2s7sRENLx3UmOMaWr8LwjKi+G7pyD5VEg5tUGrVlcrj3yxhoSYcK45KdmZ+owxppH5XxAsegX273btDTTQZz9vZ2V2IX88rydhwdaVhDGmZfCvICgrgnlPQ7ezoMtJDVu1soq/f7mOvp1aM25gwtFXMMaYZsK/gmDhS1C895j2Bt6cv5Xs/BLuHdObAOtKwhjTgvhPEJQWwvxnocd5kNiwgdDyi8v51zcbOL1nO0Z0b+tQgcYY4xv+EwQLXoSSPFefQg30/JyN7CurZNLoXg4UZowxvuVoX0NNyokToFU76DS4Qatl5hbzxrwtXHpiIr3jWztUnDHG+I7/7BFEdYQh1zZ4tSdnrEMEfn9uT+/XZIwxTYD/BMExWJldwCfLtnPDKSnER4f7uhxjjHGEBUEdDoxDHBsZwi1ndPN1OcYY4xgLgjrMWZ/D/I17ueOs7rQOC/Z1OcYY4xgLglpUVSuPpa+lS1wEvx7WxdflGGOMoywIavHR0izW7drHn87rRUiQfUXGmJbNWrkaSsqreGrGegYlxTCmf0dfl2OMMY6zIKjhtXmb2VlYauMQG2P8hgWBh71FZbwwZyMj+3QgLSXW1+UYY0yjsCDw8K9vMiipqOLPo6wrCWOM/7AgcNuyZz9v/7iVK4Ym0b19K1+XY4wxjcbRIBCRUSKyTkQyRGRSLfNvEZEVIrJMRL4XkT5O1nMkf/9qLSFBAdx1Tg9flWCMMT7hWBCISCAwGRgN9AGurKWhf1dV+6vqIODvwFNO1XMkS7flkb5iJxNP60r7qDBflGCMMT7j5B5BGpChqptUtRyYAozzXEBVCz1eRgLqYD21UlUeTV9D21ah3HRq18Z+e2OM8Tknu6FOADI9XmcBw2ouJCK3Ab8HQoCzatuQiEwEJgJ07tzZq0XOXL2LRVvy+Nuv+hEZ6j+9chtjzAE+P1msqpNVtRvwZ+C+OpZ5SVVTVTW1Xbt2XnvviqpqHvtyLd3aRXJFapLXtmuMMc2Jk0GQDXi2ronuaXWZAlzkYD2HeX9RJpty9jNpdG+CAn2eicYY4xNOtn6LgB4ikiIiIcB4YLrnAiLieYnOWGCDg/UcoqiskqdnrSctOZZzerdvrLc1xpgmx7GD4qpaKSK3A18BgcBrqrpKRB4CFqvqdOB2ETkHqADygN84VU9NL327iT1F5bx8TS/rSsIY49ccPTuqqulAeo1p93s8v9PJ96/L7sJSXv52E2MHxDO4cxtflGCMMU2GXx4Y/+esDVRWV/On807wdSnGGONzfhcEGbv38f6ibVw1rAtd4iJ9XY4xxvic3wXBY1+sJTIkiDvOtq4kjDEG/CwIfty0l1lrdvPbM7sRGxni63KMMaZJ8JsgONCVRHx0GNePSPF1OcYY02T4TRD8d8UOlmcV8PuRPQkLDvR1OcYY02T4TRBEhgRxbp8OXHxioq9LMcaYJsVvelk7s1d7zuxldxAbY0xNfrNHYIwxpnYWBMYY4+csCIwxxs9ZEBhjjJ+zIDDGGD9nQWCMMX7OgsAYY/ycBYExxvg5UVVf19AgIpIDbD3G1dsCe7xYTnNn38eh7Pv4hX0Xh2oJ30cXVW1X24xmFwTHQ0QWq2qqr+toKuz7OJR9H7+w7+JQLf37sENDxhjj5ywIjDHGz/lbELzk6wKaGPs+DmXfxy/suzhUi/4+/OocgTHGmMP52x6BMcaYGiwIjDHGz/lNEIjIKBFZJyIZIjLJ1/X4iogkichsEVktIqtE5E5f19QUiEigiPwkIp/7uhZfE5EYEZkqImtFZI2InOTrmnxFRO52/ztZKSLviUiYr2tygl8EgYgEApOB0UAf4EoR6ePbqnymEviDqvYBhgO3+fF34elOYI2vi2gingG+VNVewED89HsRkQTgDiBVVfsBgcB431blDL8IAiANyFDVTapaDkwBxvm4Jp9Q1R2qutT9fB+uf+QJvq3Kt0QkERgLvOLrWnxNRKKB04BXAVS1XFXzfVqUbwUB4SISBEQA231cjyP8JQgSgEyP11n4eeMHICLJwGBggY9L8bWngT8B1T6uoylIAXKA192Hyl4RkUhfF+ULqpoNPAFsA3YABao6w7dVOcNfgsDUICKtgI+Au1S10Nf1+IqInA/sVtUlvq6liQgCTgReUNXBwH7AL8+piUgbXEcOUoBOQKSIXO3bqpzhL0GQDSR5vE50T/NLIhKMKwTeUdWPfV2Pj40ALhSRLbgOGZ4lIm/7tiSfygKyVPXAXuJUXMHgj84BNqtqjqpWAB8DJ/u4Jkf4SxAsAnqISIqIhOA64TPdxzX5hIgIruO/a1T1KV/X42uqeo+qJqpqMq7/L75R1Rb5q68+VHUnkCkiJ7gnnQ2s9mFJvrQNGC4iEe5/N2fTQk+cB/m6gMagqpUicjvwFa4z/6+p6iofl+UrI4AJwAoRWeaedq+qpvuuJNPE/A54x/2jaRNwnY/r8QlVXSAiU4GluK62+4kW2tWEdTFhjDF+zl8ODRljjKmDBYExxvg5CwJjjPFzFgTGGOPnLAiMMcbPWRAY04hE5Azr4dQ0NRYExhjj5ywIjKmFiFwtIgtFZJmIvOger6BIRP7p7p/+axFp5152kIj8KCI/i8g0dx81iEh3EZklIstFZKmIdHNvvpVHf//vuO9aNcZnLAiMqUFEegNXACNUdRBQBVwFRAKLVbUvMBd4wL3Km8CfVXUAsMJj+jvAZFUdiKuPmh3u6YOBu3CNjdEV193exviMX3QxYUwDnQ0MARa5f6yHA7txdVP9vnuZt4GP3f33x6jqXPf0/wAfikgUkKCq0wBUtRTAvb2Fqprlfr0MSAa+d/xTGVMHCwJjDifAf1T1nkMmivxfjeWOtX+WMo/nVdi/Q+NjdmjImMN9DVwqIu0BRCRWRLrg+vdyqXuZXwPfq2oBkCcip7qnTwDmukd/yxKRi9zbCBWRiMb8EMbUl/0SMaYGVV0tIvcBM0QkAKgAbsM1SEuae95uXOcRAH4D/Nvd0Hv21jkBeFFEHnJv47JG/BjG1Jv1PmpMPYlIkaq28nUdxnibHRoyxhg/Z3sExhjj52yPwBhj/JwFgTHG+DkLAmOM8XMWBMYY4+csCIwxxs/9PxG+d8k1AC5sAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(f1s)\n",
    "plt.plot(f1s_eval)\n",
    "plt.title('model f1 micro')\n",
    "plt.ylabel('f1')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'val'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7783758381226054"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max(f1s_eval)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bidirectional"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Рекурретную сеть можно применить в прямом и обратном порядке и объединить состояния в одно (половина вектора будет про инфу до, половина про инфу после). Это назвается bidirectional. Можно превратить слой в двунаправленный параметром bidirectional=True или же отдельно подать в слой сначала прямую последовательность, а потом перевернутую (а выходы сконкатенировать). \n",
    "\n",
    "\n",
    "Для простой классификации (когда мы используем только последний стейт) первый способ не очень-то поможет (т.к. на последнем стейте у нас будет только первый стейт с обратного прохода). \n",
    "\n",
    "А вот если мы развернем последовательность (второй способ), то последним окажется стейт, соответствующий первому слову в тексте. Когда мы объединим его с вектором из прямого прогона, то получится полная информация с двух направления.\n",
    "\n",
    "\n",
    "NB: когда мы указываем bidirectional=True, то этот слой применяется и напрямую и обратно. Поэтому в отдельном применении к перевернутой последовательности я тоже использую один слой. Но кажется, что логичнее иметь один слой для прямого прогона и один для обратного."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Clf(nn.Module):\n",
    "    \n",
    "    def __init__(self, output_dim):\n",
    "        \n",
    "        super().__init__()          \n",
    "        self.VOCAB_SIZE = len(word2id)\n",
    "        self.EMB_DIM = 50\n",
    "        self.RNN_UNITS = 128\n",
    "        \n",
    "        \n",
    "        self.embedding = nn.Embedding(self.VOCAB_SIZE, self.EMB_DIM)\n",
    "        \n",
    "        # через параметр делается вот так, а в forward будет точно также как в сетке выше\n",
    "#         self.rnn = nn.LSTM(self.EMB_DIM, self.RNN_UNITS, num_layers=1, \n",
    "#                            batch_first=True, bidirectional=True)\n",
    "        \n",
    "        #через разворот входной последовательности\n",
    "        self.rnn = nn.LSTM(self.EMB_DIM, self.RNN_UNITS, num_layers=1, \n",
    "                           batch_first=True)\n",
    "        \n",
    "        # bidirectional создает 2 вектора размерности unit, соответственно на следующем слое умножаем размерность на 2\n",
    "        self.fc = nn.Linear(self.RNN_UNITS*2, output_dim)\n",
    "        \n",
    "        self.act = nn.LogSoftmax(1) \n",
    "        \n",
    "        \n",
    "    def forward(self, text):\n",
    "\n",
    "        embedded = self.embedding(text)\n",
    "        embedded_backward = torch.flip(embedded, (1, ))\n",
    "        \n",
    "        # применяем rnn слой напрямую и в обратную\n",
    "        rnn_output, (h, *_) = self.rnn(embedded)\n",
    "        rnn_output_backward, (h_backward, *_) = self.rnn(embedded_backward)\n",
    "        \n",
    "        # у нас 1 слой и для lstm мы также добавляем squeeze(0)\n",
    "        h, h_backward = h.squeeze(0), h_backward.squeeze(0)\n",
    "        \n",
    "        # конкатим последнии стейты\n",
    "        rnn_output_bidirectional = torch.cat([h, h_backward], 1) # 2 - номер dim по которому конкатим\n",
    "        output = self.act(self.fc(rnn_output_bidirectional))        \n",
    "        return output\n",
    "\n",
    "\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')  \n",
    "model = Clf(len(label2id))\n",
    "optimizer = optim.Adam(model.parameters(), lr=0.001)\n",
    "criterion = nn.NLLLoss() \n",
    "model = model.to(device)\n",
    "criterion = criterion.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "Loss: 2.3372486877441405; F1 measure: 0.2190234375\n",
      "Loss: 2.206665351986885; F1 measure: 0.26642578125\n",
      "Loss: 2.0734622244040173; F1 measure: 0.3178125\n",
      "Loss: 1.9476312279701233; F1 measure: 0.364375\n",
      "Loss: 1.8403240699768066; F1 measure: 0.4031328125\n",
      "Eval -  0.6114715038314176\n",
      "1\n",
      "Loss: 1.2095031917095185; F1 measure: 0.6300390625\n",
      "Loss: 1.166721830368042; F1 measure: 0.644375\n",
      "Loss: 1.1288194676240286; F1 measure: 0.6559505208333334\n",
      "Loss: 1.0938479445874691; F1 measure: 0.667470703125\n",
      "Loss: 1.06243725669384; F1 measure: 0.676671875\n",
      "Eval -  0.7259925766283525\n",
      "2\n",
      "Loss: 0.8171924716234207; F1 measure: 0.7472265625\n",
      "Loss: 0.8103998801112176; F1 measure: 0.7501171875\n",
      "Loss: 0.7957117237647374; F1 measure: 0.7546875\n",
      "Loss: 0.7849208126962185; F1 measure: 0.75771484375\n",
      "Loss: 0.7740571024417877; F1 measure: 0.7604765625\n",
      "Eval -  0.7663876915708813\n",
      "3\n",
      "Loss: 0.6426291182637215; F1 measure: 0.7962109375\n",
      "Loss: 0.6469307927787304; F1 measure: 0.79453125\n",
      "Loss: 0.6388481957713763; F1 measure: 0.7966927083333334\n",
      "Loss: 0.6336217307299375; F1 measure: 0.79826171875\n",
      "Loss: 0.629605474114418; F1 measure: 0.7996875\n",
      "Eval -  0.7799688697318008\n",
      "4\n",
      "Loss: 0.537582311630249; F1 measure: 0.8264453125\n",
      "Loss: 0.536359294950962; F1 measure: 0.82650390625\n",
      "Loss: 0.5350742428501447; F1 measure: 0.82734375\n",
      "Loss: 0.5335178373008966; F1 measure: 0.827783203125\n",
      "Loss: 0.5337618681192398; F1 measure: 0.8276015625\n",
      "Eval -  0.7955010775862069\n",
      "5\n",
      "Loss: 0.4572770294547081; F1 measure: 0.852265625\n",
      "Loss: 0.45972884237766265; F1 measure: 0.85099609375\n",
      "Loss: 0.4622336432337761; F1 measure: 0.8502213541666667\n",
      "Loss: 0.4588065794110298; F1 measure: 0.85115234375\n",
      "Loss: 0.4590543664097786; F1 measure: 0.8510078125\n",
      "Eval -  0.7949760536398467\n",
      "6\n",
      "Loss: 0.38768602028489113; F1 measure: 0.870703125\n",
      "Loss: 0.3906697415560484; F1 measure: 0.8723046875\n",
      "Loss: 0.391552209208409; F1 measure: 0.8726432291666667\n",
      "Loss: 0.39255832325667145; F1 measure: 0.871865234375\n",
      "Loss: 0.3920662746727467; F1 measure: 0.87175\n",
      "Eval -  0.7939685105363985\n",
      "7\n",
      "Loss: 0.3238652621209621; F1 measure: 0.895390625\n",
      "Loss: 0.3220719303935766; F1 measure: 0.89638671875\n",
      "Loss: 0.32585082347194355; F1 measure: 0.8945182291666667\n",
      "Loss: 0.3284278103709221; F1 measure: 0.89365234375\n",
      "Loss: 0.33159339654445646; F1 measure: 0.89221875\n",
      "Eval -  0.7975634578544062\n",
      "8\n",
      "Loss: 0.25968551114201543; F1 measure: 0.916953125\n",
      "Loss: 0.2593874606490135; F1 measure: 0.91634765625\n",
      "Loss: 0.26511427740255994; F1 measure: 0.9147526041666667\n",
      "Loss: 0.2678984479606152; F1 measure: 0.91419921875\n",
      "Loss: 0.2716942505538464; F1 measure: 0.9127734375\n",
      "Eval -  0.7984644396551724\n",
      "9\n",
      "Loss: 0.20780025683343412; F1 measure: 0.9365625\n",
      "Loss: 0.2123702511936426; F1 measure: 0.934375\n",
      "Loss: 0.21325475285450618; F1 measure: 0.9335286458333333\n",
      "Loss: 0.21505800604820252; F1 measure: 0.932587890625\n",
      "Loss: 0.2163568303436041; F1 measure: 0.932109375\n",
      "Eval -  0.7932297653256706\n"
     ]
    }
   ],
   "source": [
    "f1s = []\n",
    "f1s_eval = []\n",
    "\n",
    "for i in range(10):\n",
    "    print(i)\n",
    "    f1s.append(train(model, training_generator, optimizer, criterion, 100))\n",
    "    evl = evaluate(model, valid_generator, criterion)\n",
    "    print('Eval - ', evl)\n",
    "    f1s_eval.append(evl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAtnElEQVR4nO3deXxV9bnv8c+TgQRIICODJJAoILOgAamotY4oVm3Vaqu2WivtaW2tp/ecY3t7O3h7TnvObc/paWtPayvVWodatS0VKq3Woc6AYBgVBAKBQGYIQ8j03D/WDmxigBCzspO9v+/Xa7+y9lq/vfLsDVnP2s/6rd/P3B0REUlcSbEOQEREYkuJQEQkwSkRiIgkOCUCEZEEp0QgIpLglAhERBKcEoEkJDO738y+08W2W8zswqNsG2hmfzKz3Wb2u56N8j2/a7SZ7TWz5DB/jyQeJQKR9+caYDiQ6+7XmtlIM1toZjvMzM2sqKd+kbtvdfcMd2/tqX2KgBKByPs1BnjH3Vsiz9uAp4GrYxfSkcwsJdYxSN+mRCB9VqQk809mVmpm+8zsPjMbbmZ/NrMGM3vGzLKj2l9hZmvMrN7MnjeziVHbZpjZm5HX/RZI7/C7LjezlZHXvmJm07oQ37eBbwDXRUo2t7r7Lnf/KbC0p9+jmRVFvmWkRJ7nmNmvIt8+6szsD5H155lZuZn9i5ntBH5lZmlm9sNI2x2R5bSuxCjxT4lA+rqrgYuA8cCHgT8DXwPyCf7/fgnAzMYDjwBfjmxbDPzJzAaY2QDgD8CDQA7wO6LO2M1sBrAA+CyQC/wcWHi8A6W7fxP4N+C3kZLNfWG+x048CAwCJgPDgP+K2jaC4L2OAeYD/xuYDUwHTgNmAV/vZrwSZ5QIpK/7ceQsezvwd+B1d1/h7o3A74EZkXbXAYvc/a/u3gx8HxgInEVwAEwFfujuze7+OEeesc8Hfu7ur7t7q7s/AByMvK43dPU9HmJmI4FLgc+5e13kfb0Q1aQN+Ka7H3T3A8ANwN3uXunuVcC3gZvCfmPSP6h2KH3drqjlA508z4gsnwSUtW9w9zYz2waMAlqB7X7kCItlUctjgE+Z2Rej1g2I7LM3dPU9RisEat297ij7rIokknZHfD6R5d56f9LH6RuBxIsdBAd0AMzMCA6W24EKYFRkXbvRUcvbgH9196yoxyB3f6Q3Au+mbUCOmWUdZXvHYYWP+HwI3v+OEOKSfkiJQOLFY8A8M7vAzFKBrxCUd14BXgVagC+ZWaqZfZSgRt7uF8DnzOxMCww2s3lmltmdQMwsHWi/vpAWed6j3L2C4FrCT80sO/K+zj3GSx4Bvm5m+WaWR3CR+zc9HZf0T0oEEhfc/W3gRuDHQDXBRdcPu3uTuzcBHwVuBmoJric8GfXaZcBtwE+AOmBjpG13HQD2RpbXR56H4SagOfI7KgkulB/Nd4BlQCmwCngzsk4E08Q0IiKJTd8IREQSnBKBiEiCUyIQEUlwSgQiIgmu391QlpeX50VFRbEOQ0SkX1m+fHm1u+d3tq3fJYKioiKWLVsW6zBERPoVMys72jaVhkREEpwSgYhIglMiEBFJcP3uGkFnmpubKS8vp7Gx8fiN+7H09HQKCgpITU2NdSgiEkfiIhGUl5eTmZlJUVERRw4wGT/cnZqaGsrLyykuLo51OCISR+KiNNTY2Ehubm7cJgEAMyM3Nzfuv/WISO+Li0QAxHUSaJcI71FEel9clIZEROJV/f4mVm6rZ+W2ei6cOJwpo4b2+O9QIugB9fX1PPzww3z+858/oddddtllPPzww2RlZYUTmIj0KwdbWllX0cDKrXW8Vb6bldvq2Vy9DwAzyM1IUyLoq+rr6/npT3/6nkTQ0tJCSsrRP+LFixeHHZqI9FHuTlnNft4qr2fF1uCMf+2OPTS1tgGQn5nG9MIsrjmjgBmFWUwtGEpmejg9BpUIesBdd93Fu+++y/Tp00lNTSU9PZ3s7GzWr1/PO++8w1VXXcW2bdtobGzkjjvuYP78+cDh4TL27t3LpZdeytlnn80rr7zCqFGj+OMf/8jAgQNj/M5EpKdEl3hWbqvnrW311O1vBmBgajJTRw3l5jlFTC/MYnphFiOHpvfadcG4SwTf/tMa1u7Y06P7nHTSEL754clH3f69732P1atXs3LlSp5//nnmzZvH6tWrD3XzXLBgATk5ORw4cICZM2dy9dVXk5ube8Q+NmzYwCOPPMIvfvELPvaxj/HEE09w44039uj7EJHeEV3iaT/wb6nZDwQlnnHDMrho0nCmF2YzvTCL8cMzSEmOXd+duEsEfcGsWbOO6Ov/ox/9iN///vcAbNu2jQ0bNrwnERQXFzN9+nQAzjjjDLZs2dJb4YrI+9Be4ok+248u8QyLlHiuLSkMvcTTXXGXCI515t5bBg8efGj5+eef55lnnuHVV19l0KBBnHfeeZ3eC5CWlnZoOTk5mQMHwprvXETej7p9TbxVfowST0HsSjzdFXeJIBYyMzNpaGjodNvu3bvJzs5m0KBBrF+/ntdee62XoxOR7mpubWPtjj2s6Cclnu5SIugBubm5zJkzhylTpjBw4ECGDx9+aNvcuXP52c9+xsSJEzn11FOZPXt2DCMVkWNpaGxmxdZ6lm2pZVlZHSu21nOguRU4XOL52MxCphdmMa0gi4y0+DiEmrvHOoYTUlJS4h0nplm3bh0TJ06MUUS9K5Heq0jYdu5uZOmWWpaX1bF0Sy3rKvbQ5pBkQSeRkjE5lBRlc/ro7H5R4jkWM1vu7iWdbQs1nZnZXOC/gWTgl+7+vQ7bxwALgHygFrjR3cvDjElEElNbm7Oxai9Lt9SybEtw4C+vC67FDUxN5vQxWdx+/jhmFmUzY3R23Jztd0Vo79TMkoF7gIuAcmCpmS1097VRzb4P/NrdHzCz84HvAjeFFZOIJI7G5lZWbd996MC/vKyO3QeCi7p5GWnMLMrmljnFzCzKZuLIIaT2w9p+Twkz5c0CNrr7JgAzexS4EohOBJOAf4wsPwf8IcR4RCSO1e1rCko8ZbUs31JHafnuQ104T8kfzKVTRlBSlMPMomxG5wzq12WenhZmIhgFbIt6Xg6c2aHNW8BHCcpHHwEyzSzX3WuiG5nZfGA+wOjRo0MLWET6B3dnW+0BlpXVsnRLHcu21LKhci8AqcnG1FFDuWVOESVFOZwxJpucwQNiHHHfFusi2P8CfmJmNwMvAtuB1o6N3P1e4F4ILhb3ZoAiEnstrW2s39lwRH2/suEgAJnpKZSMyeaqGaMoGZPNaYVZpKcmxzji/iXMRLAdKIx6XhBZd4i77yD4RoCZZQBXu3t9iDGJSD+wv6mFFVvrD/XoebOsjn1NwTniqKyBnHVKLiVFQY+e8cMySUpSmef9CDMRLAXGmVkxQQK4HvhEdAMzywNq3b0N+CpBD6K4l5GRwd69e2MdhkifUb334KEz/WVbalm9Yw+tbY4ZTBwxhGvOKOCMohxKxmRzUpYGY+xpoSUCd28xs9uBJQTdRxe4+xozuxtY5u4LgfOA75qZE5SGvhBWPCLSN7SPzRNd5tkUGXM/LSWJ6YVZ/MMHT2FmcQ4zRmcxpI+NyxOPQr1G4O6LgcUd1n0javlx4PEwY+gNd911F4WFhXzhC0Ee+9a3vkVKSgrPPfccdXV1NDc3853vfIcrr7wyxpGK9L72+v4bm2sPXdytitT3swalUjImm+tmFlJSlMPUUUMZkJK43ThjJdYXi3ven++Cnat6dp8jpsKl3zvq5uuuu44vf/nLhxLBY489xpIlS/jSl77EkCFDqK6uZvbs2VxxxRXqsiZx70BTKyu21R062+9Y3z97bB4lRdnMKsrhlPwM1ff7gPhLBDEwY8YMKisr2bFjB1VVVWRnZzNixAjuvPNOXnzxRZKSkti+fTu7du1ixIgRsQ5XpEfV7mti2ZZalm4JzvZXb99NS6S+f+rwTD56egEzi1Xf78viLxEc48w9TNdeey2PP/44O3fu5LrrruOhhx6iqqqK5cuXk5qaSlFRUafDT4v0J+3995duCco8b2yu5d2qoL4/IDmJ0wqHMv/ck5lZlMPpY7IZOlD1/f4g/hJBjFx33XXcdtttVFdX88ILL/DYY48xbNgwUlNTee655ygrK4t1iCInrLXNWb9zD0s317K0LLhxa9eeoL4/JD2FkqIcrj6jgFlFOUwZNVT99/spJYIeMnnyZBoaGhg1ahQjR47khhtu4MMf/jBTp06lpKSECRMmxDpEkeNqbG5l5bb6SKkn6L/fcLAFgJFD0zmzOJeZxcEwDeq/Hz+UCHrQqlWHL1Ln5eXx6quvdtpO9xBIX1G/Pxif540ttSzdXMuq7btpbg1u3h8/PIMrpp/EzMiNWwXZg2IcrYRFiUAkgWyvP8CyLUFtf9mWOt7eFcys1z4+z6fPLmZmZAz+rEEanydRKBGIxKn28fff2Fx76Oat7fXB+PsZaSmcPiaby6eNZGZxDtM1Pk9Ci5tE4O5x30e/v80mJ72rqaUtavz9YKrF+sik6vmZacwqyuEz5xQzsyiHCSMy++XcuhKOuEgE6enp1NTUkJubG7fJwN2pqakhPT091qFIH9HQ2Mybkfl139hcy8pt9RxsCcbfPzlvMBdPGs7MohxmFuUwJlfj78vRxUUiKCgooLy8nKqqqliHEqr09HQKCgpiHYbESGVDI0s310Vu3Dpyft3JJw3lhjPHMKs4mzPG5JCfmRbrcKUfiYtEkJqaSnFxcazDEOkx7s7m6n0s2xL06Fm2pZYtNfsBSE9NYkZhdsLOrys9T/97RPqAtjbn7V0NvPJuzaE+/NV7gxu3sgelUlKUwyfOHM3Mohwmn6SB2aRnKRGIxEh53X5e3ljNSxtrePXdaqr3NgFQkD2Qc8blMbMoh1nF2Zycp4HZJFxKBCK9pG5fE69uquGljdW8vLGaskipJz8zjXPG5XPWKbmcNTaPURqYTXqZEoFISA40tbKsrPbQgX/Njj24B334Z5+cw81nFTFnbB7jhmWoR4/ElBKBSA9pbXNWbd8dlHs2VLO8rI6m1jZSk40Zo7O588LxzBmby7SCLFLVh1/6ECUCkW5yd96t2scr7wYH/lc31dDQGAzQNnHkED511hjmjM1jVnEOgwboT036Lv3vFDkBlXsaefndal7aUMPLG6vZuSeYY2JU1kDmTR3JnLF5fOCUXPIy1I9f+g8lApFj2NPYzOubank5UuffUBmMHJs1KJU5p+QxZ2wec8bmMjpHd+5K/6VEIBLlYEsrK7bW88rGal7aWM1b5btpbXPSU5OYWZTDNWcUMGdsHpNGDlGXTokbSgSS8DZX7+OZtbt4aWM1b2yu5UBzK0kG0wqy+IcPnsKcsXmcPiaLtBSNzinxSYlAEo67s2bHHpas2cmSNTt5Z1dQ7hk7LIOPlQRn/GeenKv5diVhKBFIQmhtc5ZtqWXJml0sWbOT7fUHSDKYWZTDNy6fxMWTh2sGLklYSgQStw62tPLyxmqWrN7FM+t2UbOviQHJSZw9Lo87LhjHBROHkavePSJKBBJfGhqbef7tKp5es5Pn11eyr6mVjLQUPjRhGJdMHs55pw7TSJ0iHegvQvq96r0HeWZtUPJ5eWMNTa1t5GUM4IrpJ3Hx5BGcdUquLvSKHIMSgfRL22r3s2TNTv6yZhfLymppcyjMGcgnPzCGS6aM4PTR2SSre6dIlygRSL/gHozXv2R1cOa/tmIPABNGZHL7+eOYO3kEE0dm6qYukW5QIpA+q63NWbGtnr+s2cnTa3ZSVrMfMzh9dDZfu2wCl0wewZjcwbEOU6TfCzURmNlc4L+BZOCX7v69DttHAw8AWZE2d7n74jBjkr6tqaWN1zbVsGTNTv66dheVDQdJTTY+cEoe8889mYsmDWdYZnqswxSJK6ElAjNLBu4BLgLKgaVmttDd10Y1+zrwmLv/j5lNAhYDRWHFJH3T/qYWXni7iiVrdvLs+koaGlsYmJrMeafmc8nkEXxowjDd3CUSojC/EcwCNrr7JgAzexS4EohOBA4MiSwPBXaEGI/0Ie7O65treeCVLfxtfSUHW9rIGpTKJZNHcMnkEZwzLo/0VPX0kV7iDt4Gba3BT2+NWm6DlHRIHQhxeg0qzEQwCtgW9bwcOLNDm28BfzGzLwKDgQs725GZzQfmA4wePbrHA5Xe09TSxlOlO7jvpc2s2bGHnMED+Pis0Vw8eTizinJI0YQt8aGtDZr3H3407YfmA9C8L/jZFPl5xPb2R9T21oORA7J3ODi39ux6/PjvKSkF0odC2pDgZ/qQyHJW1PLQoyxH2iT3zW+2sb5Y/HHgfnf/gZl9AHjQzKa4e1t0I3e/F7gXoKSkpAv/YtLX1O5r4qHXyvj1a2VUNRxk3LAMvvvRqXxkxqjeP/M/uBcq10HlGti1Bnathep3oK0lOOOzJCDys9PnXWxz6PnR2rQ/j9rnoUcyJCUHy0nJUc+7sd6SICnpxPYBUQfr9oP3/uMc3KO2tTSe+L9LSjqkDoIBg4Oz79RBkDwgKrYUSEmLek/t7886vNdO3vf7Xd9yABp3Q+MeOLjn8PLedyPP90BTQxfe48BjJIvIcnpW5PmQqMQTWR6QGcTYw8JMBNuBwqjnBZF10W4F5gK4+6tmlg7kAZUhxiW9aMOuBha8vJkn39zOwZY2zh2fz/evLebccXnhd/Vsa4XazbBrNVSujRz0V0PdlsNtBmTAsEkw/pLgQORtQKRM0F4uwCPL3mF7xzZth9t13Een+yVSjojafsSZavSZbPTPzta3vbfdkedT75MdeYBOHQQDIj8zRhxeTh0UtDla2yOeD4TUqHYhHOB6VVvr4aTQuPvIhHFoucP6A3VQX3b4Na0Hj/07Lvs+zLqtx0MPMxEsBcaZWTFBArge+ESHNluBC4D7zWwikA5UhRiT9AJ358UN1dz30mZefKeKtJQkPnr6KD49p5hxwzPD+aX7qoOD/K7IAb9yDVSuD87kIDiryx0LI6fD9Bth+CQYPhmGju7/B6BjOaFkErUeO/LgnZIWt/XxHpOUDAOzg0d3tRyMSiS735tURs/uuXijhJYI3L3FzG4HlhB0DV3g7mvM7G5gmbsvBL4C/MLM7iQo0t3s7ir99FONza38fsV2Fry0mQ2Ve8nPTOMrF43nhtljyBk8oGd+SXMjVL8dObuPPCrXwt5dh9sMzg8O8iWfDn4OnwT5E4Izz0STlAQk9dnatHSQkgYZ+cGjF1l/O+6WlJT4smXLYh2GRKlsaOTBV8t46PWt1O5rYtLIIdx6djGXnzay+2P8uEP91sNn9+21/JqNkTNWglJO/qkwfEpwwB8WOcvPGNZzb04kTpjZcncv6WxbrC8WSz+2Zsdu7ntpM396awctbc4FE4Zz69nFzD4558Tq/427IyWdqFp+5brg63C7rDHBAX/SFZGD/mTIORmS9V9Y5P3SX5GckLY259n1ldz30iZe21TLoAHJfGLWaG6ZU0xR3jGGe3CHhgqo3hD00In+uaf8cLv0ocFBftp1kTr+FBg2EdJCurYgIkoE0jX7Drbw+PJyfvXyZrbU7Oekoel89dIJXD9r9JF3/bYchJp3oaaTA37T3sPtBmRA3jgYc1ZwoB8+JTjwDxmli5IivUyJQI5pR/0BHnhlC4+8sZU9jS1ML8zixxeN59KTU0mpexfWvnTkAb++7Mhui0MKggP+9BuCn3njIG88ZI7UAV+kj1AikE6t2FrHr/6+kTVrSjnZdvDtkQ2ck11HXmMZLHkn6P/cLiU96Jp50nSY9rHgQJ87NnikZcTsPYhI1ygRSHCxtnoDrZXvsHn9CqrLVpN7oIwfJO0kdUCkh041cGBYcJCfdFXwM298cIY/tDC+++KLxDklgkTUsAuW/gK2vhaUcyJ98JOBMZ5MatJIGD4eH3sNDJ9w+Ax/YFZMwxaRcCgRJJLqjfDKj+CtR6CthYPDZ7A2rYS/7RnKupaRZIyayLxzZ3P+5AJN8yiSQJQIEkH5cnj5v2DdU5A8gNbTbuD7DRfxs9WQkmR8eNpJfPnsYqaMGhrrSEUkBpQI4pU7bHwGXvohlL0U9M8/5yvsm3Ern/v9Nv6+oZrPnF3MbeeezPAhmvFLJJEpEcSb1mZY/SS8/N/B0AxDRsEl/wanf5K6ljRuuX8pq7bv5v9dM41rSwqPvz8RiXtKBPHi4F5Y8SC8eg/s3gb5E+Gqn8HUayA5lZ27G7npvlcpq93P/9xwOhdPHhHriEWkj1Ai6O/2VsEb9waPxnoYMwfm/QDGXXzohq1NVXu56b432H2gmV9/ehazT86Nbcwi0qcoEfRXtZuCs/8VvwmGdZgwD+Z8GQpnHtFs9fbdfGrBGwA8On+2LgiLyHsoEfQ3O1YE9f+1fwym7jvtejjrS8GNXR28vqmGzzywjCEDU3nw1lmcnK+7fEXkvZQI+gN32PRc0ANo8wvBHKZnfQnO/BwMGdnpS/66dhe3P/wmhTmDePDWWYwcmoCTsohIlygR9GWtLbD2D8E3gJ2lwdywF90NZ9wcdAc9iieWl/PPT5Qy5aQh/OqWWT03O5iIxCUlgr6oaT+sfAhe+XEwmmfuOLjiJ8GAbilpx3zpL/++ie8sWsecsbn8/KYSMtL0Tywix6ajRF+yvzbo/fP6z+FALRTMgrnfhfGXHndQN3fnB395h588t5FLp4zgh9dP7/40kSKSUJQI+oK6skgPoAeheX9w4J9zB4ye3aUx+1vbnG/8cTUPvb6Vj88q5DtXTdVYQSLSZUoEsbRzVVD/X/1kcMCfdh2c9cVgxq4uampp487HVrKotIJ/OO8U/vmSU09svmARSXhKBL3NHbb8PegB9O6zwZSNs/8BZn8eho46oV3tO9jC536znL9vqOZrl01g/rmnhBOziMQ1JYLe5A4Lbw9uAhucD+f/H5h5KwzMPuFd1e9v4pb7l/LWtnr+45ppfEzjBolINykR9KblvwqSwAduD5JAavdG/dy5u5FPLnidLdX7+ekNZzB3isYNEpHuUyLoLTtWwp//BcZeCBf9325P7bi5eh833fc6dfuauP/TMznrlLyejVNEEo4SQW84UA+PfTIoB33k3m4ngTU7gnGD2hwemT+baQVZPRqmiCQmJYKwucMfvwB7tsMtf4bB3Rv5s33coMz0FB78zJmconGDRKSHKBGE7dWfwPqn4JLvQuGsbu3imbW7+MLDb1KQPZAHbz2Tk7I0bpCI9BwlgjCVvQp//SZMvCLoItoNT75Zzj89Xsrkk4Zwv8YNEpEQKBGEZW8VPH4LZI+BK3/SpTuEO1rw0mbufmotZ52Sy72f1LhBIhKO7l217CIzm2tmb5vZRjO7q5Pt/2VmKyOPd8ysPsx4ek1bKzz5mWDsoGsfOOZIoZ1xd/7zL29z91NruWTycBbcPFNJQERCE9rRxcySgXuAi4ByYKmZLXT3te1t3P3OqPZfBGaEFU+veuE/YNPzcMWPYeS0E3ppa5vzzYWr+c1rW7mupJB//cgUUpJDzdcikuDCPMLMAja6+yZ3bwIeBa48RvuPA4+EGE/v2PgsvPDvcNonYMZNJ/TSppY27nh0Bb95bSuf/eDJfO/qqUoCIhK6MOsNo4BtUc/LgTM7a2hmY4Bi4G9H2T4fmA8wevTono2yJ+3eDk/eFgwaN+8HJ3RdYH9TC5/7zZu8+E4Vd106gc99UOMGiUjv6Cunm9cDj7t7a2cb3f1edy9x95L8/PxeDq2LWpuDi8MtB+Fjv4YBg7r80vr9Tdz4y9d5aUMV/371VCUBEelVYX4j2A5Ej4RWEFnXmeuBL4QYS/ie+RZsex2uWdDpRPJHs2tPI5+87w02V+/jpzecztwpnc9BLCISljATwVJgnJkVEySA64FPdGxkZhOAbODVEGMJ17o/BTeOzbwNplzd5Zdtqd7HTQtep3ZvE7+6ZSZzxmrcIBHpfaElAndvMbPbgSVAMrDA3deY2d3AMndfGGl6PfCou3tYsYSqdhP84fNw0ulwyb92+WVrd+zhkwveoLWtjYdvm81phVnhxSgicgyhdk5398XA4g7rvtHh+bfCjCFUzY3w2KfAkuDa+487sXy7NzbXcusDS8lIS+HR+R9g7LDMcOMUETkG3aX0fjz9L7CzFD7+2+AO4i54eWM1n75/KaMi4waN0rhBIhJj3eo1ZGYa+vKt38Ly++HsO+HUuV1+2X88vZ6RQ9P53Wc/oCQgIn1Cd7uPrj1+kzhWuQ6e+jKMORs+9PUuv2xb7X7eKt/N9bNGk5vRtTKSiEjYjloaMrN/PNomIHG/ERzcG0wyMyADrrkPkrteXVu0qgKAeVPVRVRE+o5jfSP4N4JunZkdHhnHeV38coc/3QE1G4MkkHlicwUvKq3gtIKhFOZ0/WYzEZGwHet09k3gD+6+vOMGM/tMeCH1Ycvug9WPw/lfh+JzT+ilZTX7WLV9N1+7bEJIwYmIdM+xzuy3A2Vmdkcn20pCiqfv2v4mPP1VGHsRnP2VE355e1noMpWFRKSPOVYimAQMAD5tZtlmltP+AJp7J7w+4kAd/O5TMHgYfLR7k88vKq1gemEWBdkqC4lI33Ks0tDPgWeBk4HlBBeJ23lkffxzD+4c3lMRTD4/KOeEd7Gleh9rduzh6/MmhhCgiMj7c9RE4O4/An5kZv/j7t2bcDcevPIjeHsxzP13KJzZrV20l4UuVVlIRPqg49Y4EjoJlL0Cz3wbJl0FZ36227tZVFrB6aOzdAOZiPRJidkNtCv2VsLvboHsomDKyW5MPg+wqWovayv2MG/aST0bn4hID9FYQ51pa4UnboXGerjxCUgf0u1dLT7UW+jE7jkQEektSgSdef57sPlFuOInMGLK+9rVU6UVnDEmm5FDVRYSkb5JpaGONj4DL/4/mH4DnH5ik8+/Z1eVe1m/s0FDSohIn6ZEEG13OTxxGwybBJd9/33vbrFuIhORfkCJoF1rc3BxuLX5hCefP5pFpRXMLMpmxND0HghQRCQcSgTt/vpNKH8Drvwx5I1937vbsKuBt3epLCQifZ8SAcDahfDaPTDrszD5Iz2yy0WrKjDTTWQi0vcpEdS8C3/8Aow6Ay7+To/tNigL5TB8iMpCItK3JXYiaD7QYfL5AT2y23d2NbChci+XT9O3ARHp+xL7PoI//zPsWgWfeAyyRvfYbp8qDcpCc6foJjIR6fsS9xvBykfgzV/D2f8I4y/psd26O4tKd3BmcQ7DMlUWEpG+LzETwa618NSdUHQOfOh/9+iu397VwLtV+zS2kIj0G4mXCA42BJPPpw+Bq09s8vmuWFRaQZLB3MkqC4lI/5BY1wjaJ5+vfRc+uRAyh/fw7p1FpRXMPjmX/My0Ht23iEhYEusbwdJfwuonIpPPn9Pju1+/s4FN1fuYp95CItKPJE4i2L48mHx+3MUw585QfoXKQiLSHyVOIti5CoaOgo/8vFuTzx+Pu7NoVQVnnZJHbobKQiLSf4SaCMxsrpm9bWYbzeyuo7T5mJmtNbM1ZvZwaMGccTN8/rVuTT7fFWsr9rBZZSER6YdCu1hsZsnAPcBFQDmw1MwWuvvaqDbjgK8Cc9y9zsyGhRUPAKnhTQ6zqLSC5CTjEpWFRKSfCfMbwSxgo7tvcvcm4FHgyg5tbgPucfc6AHevDDGe0BwuC+WSM7hnhqkQEektYSaCUcC2qOflkXXRxgPjzexlM3vNzOaGGE9o1uzYQ1nNfg05LSL9UqzvI0gBxgHnAQXAi2Y21d3roxuZ2XxgPsDo0T03JlBPeUplIRHpx8L8RrAdKIx6XhBZF60cWOjuze6+GXiHIDEcwd3vdfcSdy/Jz88PLeDuCMpCO5gzNo9slYVEpB8KMxEsBcaZWbGZDQCuBxZ2aPMHgm8DmFkeQaloU4gx9bhV23ezrfYAl6ssJCL9VGiJwN1bgNuBJcA64DF3X2Nmd5vZFZFmS4AaM1sLPAf8k7vXhBVTGBaVVpCSZFw8uWeHqxAR6S2hXiNw98XA4g7rvhG17MA/Rh79jrvzVGkFZ4/LI2uQykIi0j8lzp3FIXirfDfb6w+ot5CI9GtKBO/DotIdpCYbF09SbyER6b+UCLqpfcjpc8blM3RQaqzDERHpNiWCblqxrZ4duxtVFhKRfk+JoJsWlVYwIDmJCyept5CI9G9KBN3Q1uYsXlXBuePzGDpQZSER6d+UCLphxbZ6KnY3ashpEYkLSgTdsKi0ggEpSVw4UWUhEen/lAhOUHtZ6IPj88lMV1lIRPo/JYIT9ObWOnbuaeRylYVEJE4oEZygpyJloQtUFhKROKFEcALay0IfOjWfjLRYT+UgItIzlAhOwLKyOiobDjJv2kmxDkVEpMcoEZyARaU7SEtJ4oIJw2IdiohIj1Ei6KLWNmfx6p186NRhDFZZSETiiBJBFy3dUktVw0HdRCYicUeJoIsWlVaQnprE+SoLiUicUSLogtY258+rKzh/gspCIhJ/lAi64PXNNVTvbWLeVPUWEpH4o0TQBYtKKxiYmsyHJuTHOhQRkR6nRHAcLa1tPL16J+dPHMagASoLiUj8USI4jtc311Kzr4nLNROZiMQpJYLjeKq0gkEDkjnvVPUWEpH4pERwDC2tbSxZs5MLJg5n4IDkWIcjIhIKJYJjeG1TLbX7mjRBvYjENSWCY1i0ageDByRz3qnqLSQi8UuJ4CiaI72FLpw0nPRUlYVEJH4pERzFq+/WULe/WWUhEYl7SgRHsai0goy0FM4dr7KQiMQ3JYJONLe28fSanVykspCIJIBQE4GZzTWzt81so5nd1cn2m82sysxWRh6fCTOernp5YzW7D6gsJCKJIbQxE8wsGbgHuAgoB5aa2UJ3X9uh6W/d/faw4uiORaUVZKalcM74vFiHIiISujC/EcwCNrr7JndvAh4Frgzx9/WIppbgJrKLJg8nLUVlIRGJf2EmglHAtqjn5ZF1HV1tZqVm9riZFXa2IzObb2bLzGxZVVVVGLEe8vLGavY0tqgsJCIJI9YXi/8EFLn7NOCvwAOdNXL3e929xN1L8vPD7cXzVGkFmekpnD1OZSERSQxhJoLtQPQZfkFk3SHuXuPuByNPfwmcEWI8x3WwpZW/rN3JxZNGqCwkIgkjzESwFBhnZsVmNgC4HlgY3cDMousvVwDrQoznuF7aUE1DYwuXa4J6EUkgofUacvcWM7sdWAIkAwvcfY2Z3Q0sc/eFwJfM7AqgBagFbg4rnq5YVFrBkPQU5oxVWUhEEkeoU265+2JgcYd134ha/irw1TBj6KrG5lb+unYXc6eMYEBKrC+diIj0Hh3xIv6+oZqGgy3MU1lIRBKMEkHEotIdDB2YqrKQiCQcJQKCstAz6yqZO3kEqcn6SEQkseioB7z4ThV7VRYSkQSlRAAsWlVB9qBUPnBKbqxDERHpdQmfCBqbW3km0ltIZSERSUQJf+R7/u0q9jW1Mm/qSbEORUQkJhI+ESxaVUHO4AHMPjkn1qGIiMREQieCA02tPLsuKAulqCwkIgkqoY9+z79dyf6mVi7XkNMiksASOhE8taqCvIwBzCpWWUhEElfCJoL9TS38bV2lykIikvAS9gj43PoqDjSrt5CISMImgkWrdpCXkaaykIgkvIRMBPsOtvC39ZVcOmUEyUkW63BERGIqIRPB39ZX0tjcprGFRERI0ESwqLSC/Mw0ZhapLCQiknCJYO/BFp57u5LLVBYSEQESMBE8u24XB1vamDdNvYVERCABE8Gi0gqGZaZRMiY71qGIiPQJCZUIGhqbef6dKi6bOpIklYVERIAESwR/W19JU0sbl6u3kIjIIQmVCJ4qrWDEkHROH62ykIhIu4RJBA2NzbzwtspCIiIdJUwieGbdLppadROZiEhHCZMIMtNSuWjScGYUZsU6FBGRPiUl1gH0lgsnDefCScNjHYaISJ+TMN8IRESkc0oEIiIJTolARCTBhZoIzGyumb1tZhvN7K5jtLvazNzMSsKMR0RE3iu0RGBmycA9wKXAJODjZjapk3aZwB3A62HFIiIiRxfmN4JZwEZ33+TuTcCjwJWdtPu/wL8DjSHGIiIiRxFmIhgFbIt6Xh5Zd4iZnQ4UuvuiY+3IzOab2TIzW1ZVVdXzkYqIJLCYXSw2syTgP4GvHK+tu9/r7iXuXpKfnx9+cCIiCSTMG8q2A4VRzwsi69plAlOA580MYASw0MyucPdlR9vp8uXLq82srJsx5QHV3XxtPNLncSR9HofpszhSPHweY462wdw9lN9oZinAO8AFBAlgKfAJd19zlPbPA//rWEmgB2Ja5u7qmRShz+NI+jwO02dxpHj/PEIrDbl7C3A7sARYBzzm7mvM7G4zuyKs3ysiIicm1LGG3H0xsLjDum8cpe15YcYiIiKdS7Q7i++NdQB9jD6PI+nzOEyfxZHi+vMI7RqBiIj0D4n2jUBERDpQIhARSXAJkwi6OgBevDOzQjN7zszWmtkaM7sj1jH1BWaWbGYrzOypWMcSa2aWZWaPm9l6M1tnZh+IdUyxYmZ3Rv5OVpvZI2aWHuuYwpAQiaCrA+AliBbgK+4+CZgNfCGBP4todxB0cxb4b+Bpd58AnEaCfi5mNgr4ElDi7lOAZOD62EYVjoRIBHR9ALy45+4V7v5mZLmB4I981LFfFd/MrACYB/wy1rHEmpkNBc4F7gNw9yZ3r49pULGVAgyM3CA7CNgR43hCkSiJ4LgD4CUiMysCZqAhwH8I/DPQFuM4+oJioAr4VaRU9kszGxzroGLB3bcD3we2AhXAbnf/S2yjCkeiJALpwMwygCeAL7v7nljHEytmdjlQ6e7LYx1LH5ECnA78j7vPAPYBCXlNzcyyCSoHxcBJwGAzuzG2UYUjURLB8QbASyhmlkqQBB5y9ydjHU+MzQGuMLMtBCXD883sN7ENKabKgXJ3b/+W+DhBYkhEFwKb3b3K3ZuBJ4GzYhxTKBIlESwFxplZsZkNILjgszDGMcWEBUO93gesc/f/jHU8sebuX3X3AncvIvh/8Td3j8uzvq5w953ANjM7NbLqAmBtDEOKpa3AbDMbFPm7uYA4vXAe6lhDfYW7t5hZ+wB4ycCCo42CmgDmADcBq8xsZWTd1yLjQokAfBF4KHLStAm4JcbxxIS7v25mjwNvEvS2W0GcDjWhISZERBJcopSGRETkKJQIREQSnBKBiEiCUyIQEUlwSgQiIglOiUCkF5nZeRrhVPoaJQIRkQSnRCDSCTO70czeMLOVZvbzyHwFe83svyLj0z9rZvmRttPN7DUzKzWz30fGqMHMxprZM2b2lpm9aWanRHafETXe/0ORu1ZFYkaJQKQDM5sIXAfMcffpQCtwAzAYWObuk4EXgG9GXvJr4F/cfRqwKmr9Q8A97n4awRg1FZH1M4AvE8yNcTLB3d4iMZMQQ0yInKALgDOApZGT9YFAJcEw1b+NtPkN8GRk/P4sd38hsv4B4HdmlgmMcvffA7h7I0Bkf2+4e3nk+UqgCHgp9HclchRKBCLvZcAD7v7VI1aa/Z8O7bo7PsvBqOVW9HcoMabSkMh7PQtcY2bDAMwsx8zGEPy9XBNp8wngJXffDdSZ2TmR9TcBL0Rmfys3s6si+0gzs0G9+SZEukpnIiIduPtaM/s68BczSwKagS8QTNIyK7KtkuA6AsCngJ9FDvTRo3XeBPzczO6O7OPaXnwbIl2m0UdFusjM9rp7RqzjEOlpKg2JiCQ4fSMQEUlw+kYgIpLglAhERBKcEoGISIJTIhARSXBKBCIiCe7/A25LM0EJ0RIAAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(f1s)\n",
    "plt.plot(f1s_eval)\n",
    "plt.title('model f1 micro')\n",
    "plt.ylabel('f1')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'val'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7984644396551724"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max(f1s_eval)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pack и Pad"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "До этого момента расхождение между torch и tf для RNN было минимальным, касающимся только логики самого языка. Проблема в том, что rnn на торче, обычно пишутся по-другому. В торче иначе подходят к обработке последовательностей разной длины.\n",
    "\n",
    "Выше мы делали общий паддинг, чтобы иметь возможность создать Tensor. Однако при общем паддинге получается, что через RNN будет пропускаться много нулевых элементов последовательности. И чем больше максимальная длина последовательностей отличается от средней, тем больше будет этих лишних нулей в тензоре. В торче можно передавать в RNN слои последовательности разной длины. Сложность в том, что в батче длины все равно должны быть одинаковые.\n",
    "\n",
    "В торче есть специальный формат для rnn, который называется packed sequence, который упаковывает последовательности в куски одинаковой длины, которые можно напрямую передавать в RNN слои.\n",
    "\n",
    "**Full Disclosure**: Не могу сказать, что я до конца понял, как это работает. Туториалов по торчу слишком мало и сам формат какой-то неоправданно усложненный. По крайней мере оно работает. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Нужно немного переписать Dataset и train и validation лупы. \n",
    "\n",
    "В датасете нужно удалить все пустые последовательности, потому что они несовместимы с пакингом. \n",
    "\n",
    "Изменения в train/evaluate значительнее, смотрите комментарии ниже."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "class Dataset(torch.utils.data.Dataset):\n",
    "\n",
    "    def __init__(self, word2id, MAX_LEN, texts, targets):\n",
    "        \n",
    "        # чтобы у нас был цельный тензор мы все еще делаем общий паддинг\n",
    "        # Можно было бы убрать ограничение MAX_LEN в 200, \n",
    "        # но обучаться будет все-таки дольше даже с пакингом, поэтому оставим так\n",
    "        self.texts = [torch.LongTensor([word2id[w] for w in self.preprocess(t) if w in word2id][:MAX_LEN]) for t in texts]\n",
    "        self.texts = torch.nn.utils.rnn.pad_sequence(self.texts, batch_first=True)\n",
    "\n",
    "        self.word2id = word2id\n",
    "        self.MAX_LEN = MAX_LEN\n",
    "        self.target = torch.LongTensor(targets)\n",
    "        \n",
    "        \n",
    "        # в texts из-за фильтрованного словаря может получиться пустая последовательность\n",
    "        # когда мы ниже будем делать pack-pad это приведет к ошибке\n",
    "        # и вообще обучаться на нулях не очень\n",
    "        # поэтому отфильтруем texts и targets с ненулевыми последовательностями в texts\n",
    "        self.target = self.target[self.texts.sum(1).bool()] #сумма по dim=1 будет ненулевая(=True) если есть 1 ненулевой индекс\n",
    "        self.texts = self.texts[self.texts.sum(1).bool()] \n",
    "        self.length = len(self.texts)\n",
    "        \n",
    "\n",
    "    def __len__(self):\n",
    "        return self.length\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        ids = self.texts[index]\n",
    "        y = self.target[index]\n",
    "\n",
    "        return ids, y\n",
    "    \n",
    "    def preprocess(self, text):\n",
    "        tokens = text.lower().split()\n",
    "        tokens = [token.strip(punctuation) for token in tokens]\n",
    "        tokens = [token for token in tokens if token]\n",
    "        return tokens\n",
    "\n",
    "\n",
    "\n",
    "def train(model, iterator, optimizer, criterion, print_every=1000):\n",
    "    \n",
    "    epoch_loss = []\n",
    "    epoch_f1 = []\n",
    "    \n",
    "    model.train()  \n",
    "\n",
    "    for i, (texts, ys) in enumerate(iterator):\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        # У нас есть тензор с общим паддингом/т.е. лишними нулями\n",
    "        # чтобы преобразовать такой тензор в оптимальный (packed) формат \n",
    "        # нам нужны изначальный длиный этих последовательностей (без нулей).\n",
    "        # Эту инфу легко восстановить из самого тензора - \n",
    "        # заменяем все значения на True/False (нули будут False)\n",
    "        # и суммируем (True равнозначен единице)\n",
    "        # Сумма True будет равна длине последовательности без нулей\n",
    "        lengths = texts.bool().sum(1)\n",
    "        \n",
    "        # теперь нужно отсортировать тензор по убыванию длин последовательностей\n",
    "        # функция .sort - вернет сами отсортированные длины и индексы, с помощью которых\n",
    "        # можно также отсортировать тензор с текстами и таргетами (чтобы они сходились)\n",
    "        length_sorted, perm_index = lengths.sort(0, descending=True)\n",
    "        texts, ys = texts[perm_index], ys[perm_index]\n",
    "        \n",
    "        \n",
    "        predictions = model(texts.to(device), length_sorted.to(device)).squeeze()  \n",
    "        loss = criterion(predictions, ys.to(device))        \n",
    "        \n",
    "        \n",
    "        \n",
    "        loss.backward()\n",
    "        optimizer.step()     \n",
    "        preds = predictions.detach().to('cpu').numpy().argmax(1).tolist()\n",
    "        y_true = ys.tolist()\n",
    "        \n",
    "        epoch_loss.append(loss.item())\n",
    "        epoch_f1.append(f1_score(y_true, preds, average=\"micro\"))\n",
    "        \n",
    "        if not (i+1) % print_every:\n",
    "            print(f'Loss: {np.mean(epoch_loss)}; F1 measure: {np.mean(epoch_f1)}')\n",
    "        \n",
    "    return np.mean(epoch_f1)\n",
    "\n",
    "\n",
    "def evaluate(model, iterator, criterion):\n",
    "    \n",
    "    epoch_loss = []\n",
    "    epoch_f1 = []\n",
    "    \n",
    "    model.eval()  \n",
    "    with torch.no_grad():\n",
    "        for texts, ys in iterator:\n",
    "            # тут все также как и в функции выше\n",
    "            lengths = texts.bool().sum(1)\n",
    "            length_sorted, perm_index = lengths.sort(0, descending=True)\n",
    "            texts, ys = texts[perm_index], ys[perm_index]\n",
    "            \n",
    "            \n",
    "            predictions = model(texts.to(device), length_sorted.to(device)).squeeze()\n",
    "            loss = criterion(predictions, ys.to(device))        \n",
    "            preds = predictions.detach().to('cpu').numpy().argmax(1).tolist()\n",
    "            y_true = ys.tolist()\n",
    "            \n",
    "\n",
    "            epoch_loss.append(loss.item())  \n",
    "            epoch_f1.append(f1_score(y_true, preds, average=\"micro\"))\n",
    "            \n",
    "    return np.mean(epoch_f1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Пересоздаем датасет с измененным классом"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "texts = data.text.values\n",
    "id2label = {i:l for i,l in enumerate(set(data.topic))}\n",
    "label2id = {l:i for i,l in id2label.items()}\n",
    "targets = [label2id[l] for l in data.topic]\n",
    "\n",
    "train_texts, valid_texts, train_targets, valid_targets = train_test_split(texts, targets, test_size=0.05)\n",
    "\n",
    "training_set = Dataset(word2id, MAX_LEN, train_texts, train_targets)\n",
    "training_generator = torch.utils.data.DataLoader(training_set, batch_size=1024, shuffle=True, )\n",
    "\n",
    "valid_set = Dataset(word2id, MAX_LEN, valid_texts, valid_targets)\n",
    "valid_generator = torch.utils.data.DataLoader(valid_set, batch_size=1024, shuffle=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Теперь самое сложное"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Определение слоев остается таким же как и раньше. Но нужно сильно поменять forward. \n",
    "\n",
    "С входящими данными нужно сделать следующее - прогнать тензор с общим паддингом через ембеддинг слой, упаковать последовательность эмбедингов, чтобы убрать нули, прогнать packed последовательность через rnn, распаковать выход rnn в тензор с общим паддингом, выбрать стейт соответствующий последнему ненулевому индексу в каждой последовательности.  \n",
    "\n",
    "Смотрите в код и комментарии, чтобы понять, где что."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Clf(nn.Module):\n",
    "    \n",
    "    def __init__(self, output_dim):\n",
    "        \n",
    "        super().__init__()          \n",
    "        self.VOCAB_SIZE = len(word2id)\n",
    "        self.EMB_DIM = 100\n",
    "        self.LSTM_UNITS = 128\n",
    "\n",
    "        \n",
    "        self.embedding = nn.Embedding(self.VOCAB_SIZE, self.EMB_DIM)\n",
    "        self.rnn = nn.LSTM(self.EMB_DIM, self.LSTM_UNITS, num_layers=1, bidirectional=True,\n",
    "                            batch_first=True) \n",
    "\n",
    "        self.fc = nn.Linear(self.LSTM_UNITS*2, output_dim)\n",
    "        self.act = nn.LogSoftmax(1) \n",
    "        \n",
    "        \n",
    "#     def embedding_masking(self, \n",
    "#                           x,\n",
    "#                           pad_mask,\n",
    "#                           value: float = 0.):\n",
    "#         x = x.masked_fill((~(pad_mask.bool())).unsqueeze(-1), value)\n",
    "#         return x\n",
    "    \n",
    "    \n",
    "    # теперь нам еще нужны длины последовательностей, добавляем их аргументом\n",
    "    def forward(self, text, lengths):\n",
    "        \n",
    "        # эмбедим как обычно\n",
    "        embedded = self.embedding(text)\n",
    "        \n",
    "        \n",
    "        # теперь используем torch.nn.utils.rnn.pack_padded_sequence\n",
    "        # чтобы упаковать последовательности\n",
    "        # для упаковывания нужны длины (lengths)\n",
    "        packed_embedded = torch.nn.utils.rnn.pack_padded_sequence(embedded,\n",
    "                                                                  lengths.cpu(), \n",
    "                                                                  batch_first=True)\n",
    "\n",
    "        # rnn слои могут работать с таким форматом сразу\n",
    "        # поэтому тут никаких изменений\n",
    "        rnn_output, (h, *_) = self.rnn(packed_embedded)\n",
    "        \n",
    "        \n",
    "        # используем pad_packed_sequence чтобы вернуть упакованные последовательности\n",
    "        # к обычным тензорам\n",
    "        # дополнительный параметр тут - максимальная длина последовательности до которой\n",
    "        # нужно добавить нулей (мы можем взять ее из изначального тензора с общим паддингом)\n",
    "        # text имеет размерность (batch_size, max_len); size(1) вытащит max_len\n",
    "        unpacked_rnn_output, _ = torch.nn.utils.rnn.pad_packed_sequence(rnn_output,\n",
    "                                                                        total_length=text.size(1),\n",
    "                                                                        batch_first=True)\n",
    "        \n",
    "        \n",
    "        # теперь самая запутанная часть!\n",
    "        # Мы восстановили тензор из запакованной последовательности\n",
    "        # и на последних состояних у нас могут быть нули (если длина последовательности меньше max_len)\n",
    "        # Поэтому для каждой последовательности нам нужно достать состояние \n",
    "        # соответствующее ее последнему ненулевому токену\n",
    "        # Получить индекс последнего ненулевого элемента не сложно - это просто длина (которая у нас уже есть) минус 1\n",
    "        # минус 1 нужен т.к. индексы начинаются с нуля\n",
    "        \n",
    "        # НО в торче нельзя достать элемент как в numpy - X[:, lengths-1]\n",
    "        # нужно создать маску и использовать функцию .gather \n",
    "        # вот тут есть визиуализация как она работает - \n",
    "        # https://stackoverflow.com/questions/50999977/what-does-the-gather-function-do-in-pytorch-in-layman-terms\n",
    "        \n",
    "        # Не уверен, что я понимаю, что происходит ниже\n",
    "        # Но оно работает правильно\n",
    "        # Для всех последовательностей в батче выбирается 1 вектор состояния,\n",
    "        # соответствующий последнему ненулевому элементу\n",
    "        \n",
    "        \n",
    "        # изначально lengths - одномерный\n",
    "        # создаем ему искусственные размерности через unsqueeze\n",
    "        # получим (256, 1, 1)\n",
    "        mask = (lengths-1).unsqueeze(1).unsqueeze(1)\n",
    "        \n",
    "        # теперь нужно расширить маску чтобы было (256, 1, 256)\n",
    "        # т.е. маска будет захватывать 1 полный вектор для всех последовательностей\n",
    "        mask = mask.expand((unpacked_rnn_output.size(0), 1, unpacked_rnn_output.size(2)))\n",
    "        \n",
    "        # теперь накладываем маску на unpacked_rnn_output\n",
    "        # результат будет трехмерный\n",
    "        # но одна размерность будет 1 и ее можно убрать через squeeze\n",
    "        # получиться (batch_size, hidden_size)\n",
    "        lh = unpacked_rnn_output.gather(1, mask).squeeze(1)\n",
    "        \n",
    "        \n",
    "        # ну и дальше все также\n",
    "        hidden = self.fc(lh)\n",
    "        outputs=self.act(hidden)\n",
    "        \n",
    "        return outputs\n",
    "\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')  \n",
    "model = Clf(len(label2id))\n",
    "\n",
    "\n",
    "optimizer = optim.Adam(model.parameters(), lr=0.001)\n",
    "criterion = nn.NLLLoss() \n",
    "model = model.to(device)\n",
    "criterion = criterion.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "Loss: 2.3831879353523253; F1 measure: 0.21904296875\n",
      "Eval -  0.3236781529017857\n",
      "1\n",
      "Loss: 1.8424859702587129; F1 measure: 0.3822265625\n",
      "Eval -  0.4840262276785714\n",
      "2\n",
      "Loss: 1.372575581073761; F1 measure: 0.566435546875\n",
      "Eval -  0.6146937779017857\n",
      "3\n",
      "Loss: 1.0586782974004745; F1 measure: 0.681552734375\n",
      "Eval -  0.6886509486607143\n",
      "4\n",
      "Loss: 0.8514716416597367; F1 measure: 0.744794921875\n",
      "Eval -  0.7217145647321428\n",
      "5\n",
      "Loss: 0.702330869436264; F1 measure: 0.786552734375\n",
      "Eval -  0.7543422154017857\n",
      "6\n",
      "Loss: 0.6021782660484314; F1 measure: 0.8147265625\n",
      "Eval -  0.7556849888392857\n",
      "7\n",
      "Loss: 0.5182842272520065; F1 measure: 0.840732421875\n",
      "Eval -  0.7697056361607143\n",
      "8\n",
      "Loss: 0.4490975624322891; F1 measure: 0.86158203125\n",
      "Eval -  0.7579520089285714\n",
      "9\n",
      "Loss: 0.39140950292348864; F1 measure: 0.88099609375\n",
      "Eval -  0.7763846261160714\n"
     ]
    }
   ],
   "source": [
    "f1s = []\n",
    "f1s_eval = []\n",
    "\n",
    "for i in range(10):\n",
    "    print(i)\n",
    "    f1s.append(train(model, training_generator, optimizer, criterion, 100))\n",
    "    evl = evaluate(model, valid_generator, criterion)\n",
    "    print('Eval - ', evl)\n",
    "    f1s_eval.append(evl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAA0cElEQVR4nO3dd3xV5f3A8c83O2SQCSEJIQHCVkAigrjFigu0LnDUVamtOKp11jqotta2tvanVq1a68CFKyoCasE6QAmyd0AgC0hCFiNkfX9/nAuEJIzEnNwk9/t+vfLKPec859zvvZDne87znPM8oqoYY4zxXX7eDsAYY4x3WSIwxhgfZ4nAGGN8nCUCY4zxcZYIjDHGx1kiMMYYH2eJwPgkEXlJRB4+wrIbRWTsQbaFisiHIlImIm+3bpSN3itFRHaIiL+b72N8jyUCY36ci4DuQKyqXiwiPUQkU0TyRURFJLW13khVN6tquKrWttYxjQFLBMb8WL2Atapa41muA2YCF3ovpAOJSIC3YzDtmyUC0255mmTuEJGlIrJTRF4Qke4i8omIVIjIZyISXa/8eBFZISKlIjJXRAbW2zZcRL737PcmENLgvc4VkcWefb8RkaOPIL6HgPuBSz1NNtep6lZVfRpY0NqfUURSPVcZAZ7lGBH5t+fqo0RE3vesP0VEckXkLhHZAvxbRIJF5O+esvme18FHEqPp/CwRmPbuQuAMoB9wHvAJcC8Qj/P/92YAEekHvA7c6tk2A/hQRIJEJAh4H3gFiAHept4Zu4gMB14EfgHEAs8CmYerKFX1AeAPwJueJpsX3PyMTXgF6AIMBroBf6u3LQHns/YCJgO/BUYBw4ChwEjgvhbGazoZSwSmvfs/z1l2HvAl8K2qLlLVSuA9YLin3KXAx6r6qapWA38BQoHjcSrAQODvqlqtqtM58Ix9MvCsqn6rqrWq+h9gj2e/tnCkn3EfEekBnAXcoKolns/1Rb0idcADqrpHVXcDlwNTVXWbqhYCDwFXuv3BTMdgbYemvdta7/XuJpbDPa8TgU17N6hqnYjkAElALZCnB46wuKne617AVSJyU711QZ5jtoUj/Yz19QS2q2rJQY5Z6Ekkex3w/Xhet9XnM+2cXRGYziIfp0IHQEQEp7LMAwqAJM+6vVLqvc4BHlHVqHo/XVT19bYIvIVygBgRiTrI9obDCh/w/eB8/nwX4jIdkCUC01m8BZwjIqeLSCBwO07zzjfAPKAGuFlEAkXkpzht5Hv9C7hBRI4TR5iInCMiES0JRERCgL39C8Ge5ValqgU4fQlPi0i053OddIhdXgfuE5F4EYnD6eR+tbXjMh2TJQLTKajqGuAK4P+AIpxO1/NUtUpVq4CfAlcD23H6E96tt28WcD3wJFACZHvKttRuYIfn9WrPshuuBKo977ENp6P8YB4GsoClwDLge886YxCbmMYYY3ybXREYY4yPs0RgjDE+zhKBMcb4OEsExhjj4zrcA2VxcXGamprq7TCMMaZDWbhwYZGqxje1zdVEICLjgCcAf+B5VX20wfZeOGO8xOPc1neFquYe6pipqalkZWW5FLExxnROIrLpYNtcaxryTJ7xFM54KIOASSIyqEGxvwAvq+rRwFTgj27FY4wxpmlu9hGMBLJVdYPngZ43gAkNygwC/ut5PaeJ7cYYY1zmZiJIwhkPZa9cz7r6luA88QlwARAhIrENDyQik0UkS0SyCgsLXQnWGGN8lbc7i38DPCkiVwP/wxkgrNE0fKr6HPAcQEZGRqNHoaurq8nNzaWysrLhpk4lJCSE5ORkAgMDvR2KMaYTcTMR5OGM/rhXsmfdPqqaj+eKQETCgQtVtbS5b5Sbm0tERASpqakcOMBk56GqFBcXk5ubS1pamrfDMcZ0Im42DS0A0kUkzTND1EQgs34BEYkTkb0x3INzB1GzVVZWEhsb22mTAICIEBsb2+mveowxbc+1ROCZzHsKMAtYBbylqitEZKqIjPcUOwVYIyJrge7AIy19v86cBPbyhc9ojGl7rvYRqOoMnLlj66+7v97r6cB0N2MwxpiOSlXZUl7JirxyVuSXc/rAbgxJ6trq7+PtzuJOobS0lGnTpvGrX/2qWfudffbZTJs2jaioKHcCM8Z0GHV1ysbinazIL/f8lLEyv5zinVUAiEBMeJAlgvaqtLSUp59+ulEiqKmpISDg4F/xjBkzDrrNGNN5VdXUsW5bBSvyy1lZr9LfWeXcNBnoL6R3i+D0gd0YnNiVwYmRDOwRSViwO1W2JYJWcPfdd7N+/XqGDRtGYGAgISEhREdHs3r1atauXcv5559PTk4OlZWV3HLLLUyePBnYP1zGjh07OOusszjhhBP45ptvSEpK4oMPPiA0NNTLn8wY82PtqqphVYHnLD+vnBUFZazdsoOq2joAugT5M7BHJBeNSGZwYlcGJUbSr3sEQQFtNyZop0sED324gpX55a16zEGJkTxw3uCDbn/00UdZvnw5ixcvZu7cuZxzzjksX758322eL774IjExMezevZtjjz2WCy+8kNjYA5+bW7duHa+//jr/+te/uOSSS3jnnXe44oorWvVzGGPcVbqrihX55SzPK9vXvLOhaCd7J4KM7hLIkKSuXHNC6r4z/dTYMPz9vHsjSKdLBO3ByJEjD7jX/x//+AfvvfceADk5Oaxbt65RIkhLS2PYsGEAjBgxgo0bN7ZVuMaYZqrfibs8v2xfE09e6f7pqRO7hjA4qSvnDU3cV+n36BrSLu/+63SJ4FBn7m0lLCxs3+u5c+fy2WefMW/ePLp06cIpp5zS5LMAwcHB+177+/uze7db850bY5qjfifuck9b/or8crbX68RNiwtjRK9ofja6175KPzosyMuRH7lOlwi8ISIigoqKiia3lZWVER0dTZcuXVi9ejXz589v4+iMMUeqdFcVq7dUsLqgnDVbK1hVUMHarRXsqteJ2697BGcM7M7gpEgGJ0YyIMG9Tty20rGjbydiY2MZM2YMQ4YMITQ0lO7du+/bNm7cOJ555hkGDhxI//79GTVqlBcjNcaAc9fO+sIdrN5S7qn4K1izpYIt5fuv1qO7BDIgIZJLMnoyKNGp9NO7tW0nblsR1UZjuLVrGRkZ2nBimlWrVjFw4EAvRdS2fOmzGvNjqSoFZZWNKvz1hTuoqXPqviB/P/p2C2dAQgQDekTQPyGSgQkRxEcEt8v2/JYSkYWqmtHUNrsiMMZ0ChWV1azdWnFAhb96SznllTX7yiRFhTIgIYKxg7rtq/BT48II9O98Z/nNYYnAGNOh1NTWsbF4F6u3lLNmi9OOv3pLObkl+2+wCA8OYEBCBOcNTWRAj0gGJETQr3sEXUNtCPemWCIwxrRbhRV7Dqjw12wtZ+3WHVTVOA9j+fsJvePCGNYzikkjU+jf3WneSYoK7VTNOm6zRGCMaRd2VdWwLLeMRTmlLNpcwqLNpWyr2LNve3xEMAMSIrhqdC8GJEQyoEcEfeLDCQn092LUnYMlAmNMm6urUzYU7WRxvUp/zdYKaj0duKmxXTi+TyxHJUcxMCGC/gkRxIYHH+aopqUsERhjXFe6q4pFOaUs3lzq+V2yrxM3IjiAYSlR/GpgH4anRDGsZzQxHehhrM7AEoEXhIeHs2PHDm+HYYwrqmvrWLOlYl8Tz+LNpWwo2gmAn0C/7hGcc3Qiw3tGMTwlij7x4fh5eawdX2eJwBjzo2wpq3Qq/JxSFm0uZWleKZXVTmduXHgQw1OiuXBEMsNTojg6OYrwDv4Ubmdk/yKt4O6776Znz57ceOONADz44IMEBAQwZ84cSkpKqK6u5uGHH2bChAlejtSYH6eyupbleWUs2lzKohynbb+gzHkaN8jfj8FJkVw2shfDUqIY3jOK5Gi7e6cj6HyJ4JO7Ycuy1j1mwlFw1qMH3XzppZdy66237ksEb731FrNmzeLmm28mMjKSoqIiRo0axfjx4+2PwnQYqsrG4l0s9lT4izaXsqqgfN8TuT1jQjk2NcbTrh/FoMRIggPsDp6OqPMlAi8YPnw427ZtIz8/n8LCQqKjo0lISODXv/41//vf//Dz8yMvL4+tW7eSkJDg7XCNaVJtnbIiv4yvs4v57odiFueUUrKrGoCwIH+G9oziFyf3ZnjPaIalRBFnd/F0Gp0vERzizN1NF198MdOnT2fLli1ceumlvPbaaxQWFrJw4UICAwNJTU1tcvhpY7xFVfmhaCdfZxfxdXYx8zYUU7bbqfjTu4Xzk0EJDE+JYnhKNH27hXt98hTjns6XCLzk0ksv5frrr6eoqIgvvviCt956i27duhEYGMicOXPYtGmTt0M0hm3llXy9voiv1hXzzfqife37SVGhnDm4O2P6xnF8nzjiI+xs35e4mghEZBzwBOAPPK+qjzbYngL8B4jylLlbVTvkjO6DBw+moqKCpKQkevToweWXX855553HUUcdRUZGBgMGDPB2iMYHlVdWM399Md+sL+ar7CKytzm3LUd3CeT4PnEc3zeWE/rGkRLTxfqvfJhriUBE/IGngDOAXGCBiGSq6sp6xe4D3lLVf4rIIGAGkOpWTG5btmx/J3VcXBzz5s1rspw9Q2DcUlldy/ebS/Y19yzNLaVOITTQn2PTYrgkI5nj+8QxqEek3btv9nHzimAkkK2qGwBE5A1gAlA/ESgQ6XndFch3MR5jOp36HbzfrC/iux+2s6emDn8/YVjPKKac2pcxfeMYlhJld/SYg3IzESQBOfWWc4HjGpR5EJgtIjcBYcDYpg4kIpOByQApKSmtHqgxHcWhOnj7d4/g8uN6MaZvLCPTYogIsSGXzZHxdmfxJOAlVf2riIwGXhGRIapaV7+Qqj4HPAfODGVNHUhVO30bZ0ebTc60jr0dvF9nF/N1dtMdvKP7xNItIsTLkZqOys1EkAf0rLec7FlX33XAOABVnSciIUAcsK05bxQSEkJxcTGxsbGdNhmoKsXFxYSE2B97Z1deWc23G7Z7zvqLWOfp4I3qEsgYTwfvmD5x9Iq1Dl7TOtxMBAuAdBFJw0kAE4HLGpTZDJwOvCQiA4EQoLC5b5ScnExubi6Fhc3etUMJCQkhOTnZ22EYF+SV7ubDJfnMWrGFJTkHdvBeNCKZMX2tg9e4x7VEoKo1IjIFmIVza+iLqrpCRKYCWaqaCdwO/EtEfo3TcXy1tqD9IzAwkLS0tNYM3xjXFe/Yw4xlBWQuyWfBxhIAhiZ3ZcqpfTm+bxzDrYPXtBHpaO3OGRkZmpWV5e0wjGmRispqZq/YSuaSfL7KLqK2TunXPZwJw5I47+hEUmK7eDtE00mJyEJVzWhqm7c7i43p9Cqra5m7ZhuZS/L5fNU29tTUkRwdyi9O6s34YYkMSIg8/EGM76qthuL1ULgKegyFmN6t/haWCIxxQU1tHd+sLyZzST6zlm+hYk8NceFBTBqZwnlDEzkmJco6es2Bamtg+wanwt+2ev/v4myoc24RZtyjMOqXrf7WlgiMaSWqyvebS8hcnM/Hywoo2lFFRHAA44YkMH5YIqN7xxLg7+ftMNu/ujrYsQVKNoHWQmQiRPSAwFBvR9Y6amug5AfYtgoKV+//XbRuf4WPQHQqdBsI/cdB/EDoNgDi+rkSkiUCY34EVWX1lgoyl+Tz4ZJ8ckt2Exzgx9iB3TlvaCKn9I8nJNA6fA+gCjuLoHQzlG50KvzSTc5yySYoy4Haqsb7hUZDZJKTFCJ71Hud5CxH9HDKtJcrrbpaKNnoqej3nuV7KvzaPfvLRfVyKvz0nzi/4z0VflDb9RdZIjCmBTYX7yJzSR4fLM5n3bYd+PsJJ6bHcdsZ/ThjUHd7qnd3qVO5l3gq+ANeb4bqnQeW7xILUSnOJFADz3VeR6WCnz9UFEB5HpQX7H9dsAR2NvG4UUCoJykkOlcSByQMz7qwbuDfilXf3gq//tn9ttVQtPbACr9rinNW3+e0/RV+fH8ICmu9WFrIEoExR2hbeSUfLS3ggyX5LMkpBWBkagy/P38IZw9JINaXJmqp2tlEJe/5KdkMe8oOLB8c6Zz5xvaBPqc6r6N7eSr8FAiOaH4MNVVOE1K5JzlUFEB5vvNTUQA586FiS+OrC/GD8O77m5z2XVEk7k8WET0an5HX1Tmf74AKf5VT4dfUm2uka0+nku99sqfCH+hU+MHhzf+MbcQSgTGHULarmpkrCvhgcT7zNxRTpzA4MZJ7zhrAuUMTSYrqJO3WDdXsgdIcp+lmb5NN/bP6XUUHlg8I3V+x9xxVr5L3VPghUa3fZBMQtD+RHExdHewqhor8BgnD87o4G374H+wpb7xvSJQnKSTAru1OhV+9a//2yCSnwk87yfndbaDTpBPS8e4Cs0RgTAO7q2r5bNVWPliczxdrt1Fdq6TFhTHltHTGD02kbzcXzuwqy2D9fyFngXMGW1fjdJTW1Xl+1zb4XVevTO3Byzba72Blaw5cV79JA8AvEKJ6OhX7vqabXk6HZlQKhMW3n7b5+vz8IDze+ekx9ODl9uw4sAmqfsKoyHf6HkZcvb/Cj+8PIV3b7GO4zRKBMUBVTR1fZRfyweJ8Pl25lV1VtSREhnDV6FQmDEtiSFJk69/uWZQNa2fCulmw6RunMg4IhcAQEH+nfdwvwPPab/+6fb/9GpTxvA4Iblz2gP0a7H/Ae3nWBYY5FXx0L6fCj0hw1ndWweEQnA5x6d6OxCssERifVlldy8vzNvLMFxvYvrOKqC6BTBiWxIRhiYxMjWndsX1qq50Kf+0sJwFsX++s7zYIjr8J+o2D5GM7d4Vr2iVLBMYn1dYp7y3K4/HZa8gvq+SkfvFcNboXJ6bHExTQivf67yyG7E+dij/7c6ct2j/IaVce9UvnlsHoXq33fsa0gCUC41NUlblrC/nTJ6tZvaWCo5K68peLh3J837jWegPYttKp+NfOgpzvAHXuUhl8vnPWn3Zyu76DxPgeSwTGZyzJKeXRT1Yzb0MxKTFd+L9JwznnqB4/vvmnuhI2frm/8i/zTMyXOBxOuRv6nQkJQ512fmPaIUsEptPbWLSTP89ew8dLC4gJC+LB8wZx2XG9flwTUHmB08m7dhZsmOvcVhjYxXlY6OQ7nSafiIRW+wzGuMkSgem0inbs4f8+X8dr324m0N+Pm0/ry/Un9W7ZU791dVCwaH9Hb8ESZ33XFBh+hXPW3+sE544fYzoYSwSm09m5p4bnv/yB5/63nsqaOiYe25NbTk+nW2QzK+k9Fc7Z/tqZsHa2M6SB+EHySBj7oNPeHz+gfd4/b0wzWCIwnUZ1bR1vLMjhic/WUbRjD+MGJ3DHuP70iW9Gx2zJxv1n/Ru/ch7uCu4KfU93Kv6+YyEs1rXPYIw3WCIwHZ6qMnP5Fv48aw0binZybGo0z145ghG9og+/c20N5C6AtZ84CaBwtbM+Nh1GTnYq/5RR4O/jg8iZTs0SgenQvt1QzB8/Wc3inFLSu4Xz/M8yOH1gt0M/BVxZ5tzTv3YmrJsNu0ucp2p7jYFjrnLa+2P7tN2HMMbLLBGYDmnt1gr+9MlqPl+9jYTIEB678Gh+ekzSwSd+KdkIa2bCmhme4RyqITQG0s90Jv7oc1qnGjvGmOawRGA6lIKy3Tw+ey3vfJ9LWHAAd47rzzXHpxEa1GBYhrpayM1ymnzWzHQmBgGI6w+jfwX9zoKeI204B2OwRGA6iLLd1fxz7nr+/fUPqMK1Y9K48dS+RIcF7S+0p8IZwXONZyC3XcVOk0/KaDjmD057vzX5GNOIq4lARMYBTwD+wPOq+miD7X8DTvUsdgG6qWqUmzGZjqWyupZX5m3iyTnZlFdWc/6wJG47ox89YzyThpRudir+tZ/sv8snJArSz9h/l09olDc/gjHtnmuJQET8gaeAM4BcYIGIZKrqyr1lVPXX9crfBAx3Kx7TsdTWKR8szuOvs9eSV7qbk/rFc9e4/gxOiIC8hbDI0+SzbYWzQ2xf5y6f/mc5E6O05lSExnRybv61jASyVXUDgIi8AUwAVh6k/CTgARfjMR2AqvLF2kIe9QwKNyQpkr9M6MNolsJ3/3GafHYWOmPop4yGnzzstPfH9fV26MZ0WG4mgiQgp95yLnBcUwVFpBeQBvz3INsnA5MBUlIOMS2d6dCW5jqDwn2zvpiMqJ1kHreZo3bMQ6Z/6cyYFdwV0sc6FX/f06FLjLdDNqZTaC/XzxOB6apa29RGVX0OeA4gIyND2zIw475NxTv5y8xVbF7+NeeGLOHJuGXE7FgLS4CY3nDsz51bPFNG24NdxrjAzUSQB/Sst5zsWdeUicCNLsZi2qGa2jrefONlAte8z/2yiPjgUhQ/JGYUjL7C0+STbmP5GOMyNxPBAiBdRNJwEsBE4LKGhURkABANzHMxFtPOVJUXsvS5n3P5jrlUBoQ5TT2Dz0XSz7AmH2PamGuJQFVrRGQKMAvn9tEXVXWFiEwFslQ101N0IvCGqlqTj4+oWvYBu9+7maNrK8jqexMZk+6HgKDD72iMcYWrfQSqOgOY0WDd/Q2WH3QzBtOO7NpOzcd3ErTibdbWpbLxxOc594wzvB2VMT6vvXQWm85uzUzqMm+GnUU8XnMRKeN/y0Uje3s7KmMMYJOoGnftLoX3fgmvX8rmylAuqP49fS6aaknAmHbErgiMe9Z9Bpk3oTu28lboRB4qP4fHJx3LuCE9vB2ZMaYeSwSm9VWWw+zfwvcvUxPbn9vDf8Mn2xP55xXHcPrA7t6OzhjTgCUC07o2zIUPpkB5HjuPncLFq09hQ2kNz1+VwUn94r0dnTGmCZYITOvYswM+vR+yXoDYdAov+ZBLPq5ha3klL10zklG9bZ5fY9orSwTmx9v4Fbz/K2dI6NFTyB1+GxP/vYSyXdW8ct1IRvSyB8SMac8sEZiWq9oJn0+Fb5+B6DS45hN+CDuay/41n11Vtbx2/XEcnRzl7SiNMYdhicC0zOb58P4vYfsGGPkLGPsA60rquOzZedTWKa9fP4pBiZHejtIYcwQsEZjmqd4N/30Y5j0FUT3hqo8g7URW5pdz5Qvf4ucnvDl5FOndI7wdqTHmCFkiMEcuNwveuwGK10HGtXDGVAiOYGluKVe+8B1dgvyZdv0o0uLCvB2pMaYZLBGYw6vZA3P/CF8/ARGJcOV70Oc0ABZuKuHqF7+ja5dAXr9+1P65hI0xHYYlAnNoed87dwQVroLhV8KZj0BIVwDmbyjm2pcW0C0imGnXjyIxKtTLwRpjWsISgWlaTRX87zH48nEI7waXT4f0/SOFfrmukOtfziI5ugvTfn4c3SJDvBisMebHsERgGitY6twRtHU5DJ0E4/4IodH7Nn++aiu/fO17eseF8erPjyMuPNiLwRpjfixLBGa/2mr46m/wxZ8gNAYmvg4Dzj6gyMzlBdz0+iIGJETyynUjiepiE8oY09FZIjCOrSvh/RugYAkcdTGc9VijKSM/WJzHbW8tYWhyV166diSRITaRvDGdgSUCX1dbA988AXMfheBIuORlGDShUbG3s3K4852ljEyN4YWrjyU82P7rGNNZ2F+zLytc4/QF5C10Kv9zHoewuEbFXvt2E799bzknpsfx3JUZhAb5eyFYY4xbLBH4qgXPw8x7IagLXPQiDP4piDQq9uJXPzD1o5WcNqAbT19+DCGBlgSM6WwsEfii5e/Ax7dD3zNgwlMQ0fRkMf+cu54/zVzNuMEJ/GPScIICbGZTYzojSwS+Jneh84BYymiY+BoENL71U1V54vN1/P2zdYwfmsjjlwwlwN+SgDGdlat/3SIyTkTWiEi2iNx9kDKXiMhKEVkhItPcjMfnleXC6xMhvDtc+upBk8Bjs9bw98/WcdGIZP526TBLAsZ0cq5dEYiIP/AUcAaQCywQkUxVXVmvTDpwDzBGVUtEpJtb8fi8PTtg2kSoqYSrMpvsFFZVpn60kn9/vZHLjkvh4QlD8PNr3G9gjOlc3DzVGwlkq+oGVa0C3gAa3pd4PfCUqpYAqOo2F+PxXXW18O71sG0FXPRv6DawcZE65b73l/PvrzdyzZhUHjnfkoAxvsLNRJAE5NRbzvWsq68f0E9EvhaR+SIyrqkDichkEckSkazCwkKXwu3EPnsQ1syAcY9C+thGm2vrlDvfWcpr327mhpP7cP+5g5Am7iAyxnRO3m78DQDSgVOAScC/RCSqYSFVfU5VM1Q1Iz4+vm0j7OgWvQrf/AMyroORkxttrqmt49dvLmb6wlxuHZvOXeP6WxIwxse4mQjygJ71lpM96+rLBTJVtVpVfwDW4iQG0xo2fgUf3gq9T4Wz/tToOYGqmjqmTFtE5pJ87hzXn1vH9rMkYIwPcjMRLADSRSRNRIKAiUBmgzLv41wNICJxOE1FG1yMyXcUr4c3r4CYNLj4JfA/cFygyupabnh1ITNXbOF35w7iV6f09U6cxhivcy0RqGoNMAWYBawC3lLVFSIyVUTGe4rNAopFZCUwB7hDVYvdisln7C51bhMFmPQGhEYdsFlVuen1Rfx39TYePn8I152Q1uYhGmPaD1cfKFPVGcCMBuvur/dagds8P6Y11FbD21fD9h/gZ+9DbJ9GRT5eVsCnK7dy79kDuGJUrzYP0RjTvtiTxZ2JKnxyF2yY4wwdkXpCoyIVldVM/XAlRyV15boTenshSGNMe2OJoDP57jnIegGOvxmGX9Fkkb/OXkvhjj08f1UG/vacgDEG798+alrLus9g5t3Q/2wY+2CTRZbllvHyvI1cOaoXRydHtWl4xpj2yxJBZ7BtFUy/BroNhp/+C/waDxVdW6f89v1lxIYH85sz+3shSGNMe9WiRCAi4a0diGmhnUUw7VIIDIXL3oDgpv9pXvt2E0tzy/jduYNsikljzAFaekWw8vBFjOtq9sAbl8OOrc5E812Tmyy2raKSP89cwwl94zjv6B5tHKQxpr07aGexiBzslk4B7IrA21Thw1sgZ74zw1jyiIMWffijVeypreP35w+xJ4eNMY0c6orgD0A0ENHgJ/ww+5m28NXfYMnrcMq9MOTCgxb7cl0hmUvy+dUpfUiLC2vDAI0xHcWhbh/9HnhfVRc23CAiP3cvJHNYKzPh84dgyEVw8p0HLVZZXcvv3l9OWlwYN5zc+MEyY4yBQ5/Z5wGbROSWJrZluBSPOZz8xfDeLyApAyY82eSE83s988V6Nhbv4vcThtik88aYgzpUIhgEBAHXiki0iMTs/QGq2yY8c4DyfGcModAYmDjNuVPoIH4o2snTc9YzfmgiJ6Q3no3MGGP2OlTT0LPA50BvYCFOJ/Fe6llv2krVLnh9EuypgGtnQUT3gxZVVX73/nKCA/2479zGs5EZY0x9B70iUNV/qOpA4EVV7a2qafV+LAm0pbo6pzmoYAlc+DwkDDlk8cwl+XyVXcSdZ/anW0RIGwVpjOmoDjvWkKr+si0CMYcw5xFYlQk/eRj6n3XIomW7q/n9R6sYmtyVy46zkUWNMYdng861d0vehC//Asf8DEZPOWzxv85ew/ade3jpmmNtUDljzBGx5wHas83zIXMKpJ4IZ//1kHcIASzJKeWV+Zv42ehUhiR1baMgjTEdnSWC9qpkkzN8RNdkuORlCAg6ZPGa2jrufW8Z8eHB3P6Tfm0UpDGmM7BE0B5VljsDydVVw2VvQZeYw+7yyvxNrMgv54HzBhNhg8oZY5rB+gjam9oamH4tFK2FK9+FuPTD7rK1vJK/zl7Lyf3iOfuohDYI0hjTmVgiaG9m3wfZn8K5f4PepxzRLlM/Wkl1bR1TJwy2QeWMMc1mTUPtSdaL8O0/4bhfQsa1R7TL3DXb+HhpAVNO7UuvWBtUzhjTfK4mAhEZJyJrRCRbRO5uYvvVIlIoIos9P747mN2GufDxbyD9J3DmI0e0S2V1Lfd/sILe8WFMPtme8TPGtIxrTUMi4g88BZwB5AILRCRTVRtOavOmqh7+BvnOrGgdvPUziO8PF77Q5FSTTXl6Tjabt+9i2vXHERxgg8oZY1rGzSuCkUC2qm5Q1SrgDWCCi+/XMe3aDtMuAb9AmPQGhEQe0W7Z23bwzy/Wc8HwJI7vY4PKGWNazs1EkATk1FvO9axr6EIRWSoi00WkZ1MHEpHJIpIlIlmFhYVuxOodNVXOlUBZLkx8DaKPbEiIvYPKhQb6c+/ZNqicMebH8XZn8YdAqqoeDXwK/KepQqr6nKpmqGpGfHx8mwboGlX4+DbY+CWMfxJSRh3xru8vzmPehmLuOmsA8RHBLgZpjPEFbiaCPKD+GX6yZ90+qlqsqns8i88DB594t7OZ9yQsegVO/A0MvfSIdyvbVc0jH69ieEoUk45NcTFAY4yvcDMRLADSRSRNRIKAiUBm/QIi0qPe4nhglYvxtB9rPoHZv4NBE+DU3zZr18dmraZkVzWPnH8UfjaonDGmFbh215Cq1ojIFGAW4I8zr8EKEZkKZKlqJnCziIwHaoDtwNVuxdNubFkO7/wcegyF858BvyPPxd9vLmHad5u5dkwagxKPrFPZGGMOR1TV2zE0S0ZGhmZlZXk7jJap2gXPnghVO+H6ORDZ4/D7eNTU1nHek19TsrOKz24/mfBgeyjcGHPkRGShqjY537zVJm3p84egOBt+9kGzkgDAf+ZtYlVBOc9ccYwlAWNMq/L2XUO+Y8Nc+PYZGPmLIx5DaK+Cst08PnsNpw3oxpmDbVA5Y0zrskTQFirL4P0bIbYvjH2w2btP/XAltao8NN4GlTPGtD5rY2gLn9wNFflw3acQ1KVZu/539VY+Wb6FO87sT8+Y5u1rjDFHwq4I3LbqI1gyDU64DZKb7Kc5qN1VzqByfbuFc/2JNqicMcYddkXgph2F8OEtkHAUnHxXs3d/cs46ckt28+bkUQQFWM42xrjDEoFbVOGjW2FPOVzw4WHnHG5o3dYKnvvfBi4akcxxvWPdidEYY7CmIfcsfRNWfwSn3QfdBzVrV1Xlt+8vp0tQAPecNcClAI0xxmGJwA1luTDjDkgZDaObP9XCO9/n8d0P27nnrAHEhtugcsYYd1kiaG11dfD+r6CuFs5/+ognmdmrdFcVf5ixihG9orkko8lRuY0xplVZH0FrW/A8/PCFM/l8TPPv9PnTzNWU7a7mkQuG2KByxpg2YVcErakoGz69H/qOhRHXNHv3hZu28/p3Ofz8hDQGJNigcsaYtmGJoLXU1sB7v4CAYGeimWY+AVxdW8dv31tOYtcQbj493aUgjTGmMWsaai1f/x3yspzJ55s5oBzAS19vZPWWCp67cgRhNqicMaYN2RVBayhYCnMfhcEXwJALm717Xulu/vbZWsYO7M5PbFA5Y0wbs0TwY9XscZqEusTAOY83u0kI4KHMFajCg+Ob97yBMca0BksEP9acP8C2lTD+/5xk0EyfrdzK7JVbuWVsOsnRNqicMabtWSL4MTbPh6+fgGN+Bv3ObPbuu6pqeCBzBf26h3PdCWkuBGiMMYdnvZIttWcHvHcDRPWEM//QokP84/Ns8kp38/YNown0t5xsjPEOSwQt9envoGQjXP0xBEc0e/c1Wyp4/ssNXJrRk2NTm9+kZIwxrcVOQ1si+zPIehFG3wipY5q9e12dct/7y4gICeBuG1TOGONllgiaa3cJfDAF4gfAab9r0SGmL8xlwcYS7jl7INFhzRue2hhjWpuriUBExonIGhHJFpG7D1HuQhFREWneFF7eMOMO2FkIFzwDgSHN3n37zir++MkqRqbGcNExyS4EaIwxzeNaIhARf+Ap4CxgEDBJRBrdKC8iEcAtwLduxdJqVrwHy96Gk+6ExOEtOsQfZqyiorKGh21QOWNMO+HmFcFIIFtVN6hqFfAGMKGJcr8H/gRUuhjLj1exFT66zUkAJ97WokPMWbON6QtzueHkPvTr3vwOZmOMcYObiSAJyKm3nOtZt4+IHAP0VNWPD3UgEZksIlkiklVYWNj6kR6OKnx4M1TvggueBf/AZh+ivLKae99dRr/u4dx0el8XgjTGmJbxWmexiPgBjwO3H66sqj6nqhmqmhEfH+9+cA0tegXWzoTTH4D4/i06xB8+XsXW8kr+fNFQggOaN1mNMca4yc1EkAfUn2Ir2bNurwhgCDBXRDYCo4DMdtdhXLIJZt4DqSfCcTe06BBfrivkjQU5TD6pD0N7RrVufMYY8yO5mQgWAOkikiYiQcBEIHPvRlUtU9U4VU1V1VRgPjBeVbNcjKl59k47iXimnWz+17VjTw13v7OMPvFh3DrW5hkwxrQ/riUCVa0BpgCzgFXAW6q6QkSmish4t963VX37T9j0FYz7I0SltOgQj36yivyy3Tx20VBCAq1JyBjT/rg6xISqzgBmNFh3/0HKnuJmLM22bTV89hD0OwuGX9GiQ3yTXcSr8zdz/YlpjOgV3coBGmNM67Ani5tSW+3MMRAUBuc90aI5BnbuqeGud5eSFhfG7T9pWQezMca0BRt0rilf/hUKFsMlL0NE9xYd4s+z1pBbspu3fjHamoSMMe2aXRE0lPc9fPEYHHUJDGrq+bfD++6H7bz0zUauGp1qI4saY9o9SwT1Ve925hgI7w5nP9aiQ+yuquXO6UtIienCneOsScgY0/5Z01B9n/8eitbAFe9CaMs6d/8yew0bi3fx+vWj6BJkX68xpv2zK4K9Nn4F85+GY38OfU9v0SEWbtrOi1//wJWjejG6T2wrB2iMMe6wRABQWQ7v/xJi0uCMqS07RHUtd0xfSmLXUJtsxhjToVjbBcCse6EsF66Z6dwy2gJ/+2wtGwp38up1xxEWbF+rMabjsCuCNTOdQeXG3AIpx7XoEIs2l/Cv/21g0sienJAe18oBGmOMu3w7EewshsyboPsQOOWeFh1iT00td05fSkJkCPeePbCVAzTGGPf5bhuGKnz8a2cO4ivfhYDgFh3mH5+vY922Hfzn2pFEhDR/ngJjjPE2370iWDYdVn4Ap94DCUe17BC5ZTzzxQYuHpHMyf28ME+CMca0At9MBOX5MON2SB4Jx9/SokNU1dRxx/QlxIUHcd+5jaZiNsaYDsP3moZU4YMpzsByFzwD/i37Cp6ck83qLRW8cFUGXUOtScgY03H5XiLIehHWfw5n/wVi+7ToECvyy3h6TjY/HZ7E6QNbNiidMca0F77VNFS8HmbfB71PhYzrWnSI6to67nh7KdFhQdx/njUJGWM6Pt+5Iqirdaad9AuECU+1aNpJgGfmrmdlQTnPXjmCqC5BrRykMca0Pd9JBPOehJz5cMFz0DWpRYdYvaWcf/x3HecNTeTMwQmtHKAxxniH7ySCAedC1S44+pIW7V7jaRKKDAnkofGDWzk4Y4zxHt9JBLF9nGcGWui5LzewLK+Mpy8/hpgwaxIyxnQevtVZ3ELrtlbw90/XcfZRCZx9VA9vh2OMMa3KEsFh1NYpd0xfSliwP1MnDPF2OMYY0+pcTQQiMk5E1ohItojc3cT2G0RkmYgsFpGvRKTd3Y/5wlcbWJxTyoPjBxMX3rLxiIwxpj1zLRGIiD/wFHAWMAiY1ERFP01Vj1LVYcBjwONuxdMSGwp38NfZazljUHfGD030djjGGOMKN68IRgLZqrpBVauAN4AJ9Quoanm9xTBAXYynWWrrlDunLyUk0J9Hzh+CiHg7JGOMcYWbdw0lATn1lnOBRjO/iMiNwG1AEHBaUwcSkcnAZICUlJRWD7Qp//lmI1mbSnj8kqF0iwxpk/c0xhhv8Hpnsao+pap9gLuA+w5S5jlVzVDVjPh494d73li0k8dmrea0Ad24YHjLHj4zxpiOws1EkAf0rLec7Fl3MG8A57sYzxGpq1PufGcpgf5+/OGCo6xJyBjT6bmZCBYA6SKSJiJBwEQgs34BEUmvt3gOsM7FeI7Iq99u4rsftvO7cwaR0NWahIwxnZ9rfQSqWiMiU4BZgD/woqquEJGpQJaqZgJTRGQsUA2UAFe5Fc+RyNm+i0c/Wc1J/eK5OCPZm6EYY0ybcXWICVWdAcxosO7+eq9bNj2YC1SVu95Zip8Ij/7UmoSMMb7D653F7cW07zbzzfpi7j17IIlRod4Oxxhj2owlAiCvdDd/nLGaMX1jmTSy5+F3MMaYTsTnE4Gqcvc7S6lT5dGfHm1NQsYYn+PzieDtrFy+XFfEPWcNoGdMF2+HY4wxbc6nE0FB2W5+/9FKRvWO4fLjenk7HGOM8QqfTQSqyr3vLqOmTvnThUfj52dNQsYY3+SzieDd7/OYs6aQO87sT6/YMG+HY4wxXuOTiWBbeSUPfbiCY1Ojufr4VG+HY4wxXuVziUBVufe95eypqeOxi4Zak5Axxuf5XCLIXJLPZ6u28puf9CctzpqEjDHGpxJBYcUeHshcwfCUKK49Ic3b4RhjTLvgU4ng/g+Ws6uqlj9fdDT+1iRkjDGADyWCj5cW8MnyLfx6bD/6dovwdjjGGNNu+EwiCA8J4IxB3bn+RGsSMsaY+lwdhro9OblfPCf3c3+aS2OM6Wh85orAGGNM0ywRGGOMj7NEYIwxPs4SgTHG+DhLBMYY4+MsERhjjI+zRGCMMT7OEoExxvg4UVVvx9AsIlIIbGrh7nFAUSuG09HZ93Eg+z72s+/iQJ3h++ilqk0+VdvhEsGPISJZqprh7TjaC/s+DmTfx372XRyos38f1jRkjDE+zhKBMcb4OF9LBM95O4B2xr6PA9n3sZ99Fwfq1N+HT/URGGOMaczXrgiMMcY0YInAGGN8nM8kAhEZJyJrRCRbRO72djzeIiI9RWSOiKwUkRUicou3Y2oPRMRfRBaJyEfejsXbRCRKRKaLyGoRWSUio70dk7eIyK89fyfLReR1EQnxdkxu8IlEICL+wFPAWcAgYJKIDPJuVF5TA9yuqoOAUcCNPvxd1HcLsMrbQbQTTwAzVXUAMBQf/V5EJAm4GchQ1SGAPzDRu1G5wycSATASyFbVDapaBbwBTPByTF6hqgWq+r3ndQXOH3mSd6PyLhFJBs4Bnvd2LN4mIl2Bk4AXAFS1SlVLvRqUdwUAoSISAHQB8r0cjyt8JREkATn1lnPx8coPQERSgeHAt14Oxdv+DtwJ1Hk5jvYgDSgE/u1pKnteRMK8HZQ3qGoe8BdgM1AAlKnqbO9G5Q5fSQSmAREJB94BblXVcm/H4y0ici6wTVUXejuWdiIAOAb4p6oOB3YCPtmnJiLROC0HaUAiECYiV3g3Knf4SiLIA3rWW072rPNJIhKIkwReU9V3vR2Pl40BxovIRpwmw9NE5FXvhuRVuUCuqu69SpyOkxh80VjgB1UtVNVq4F3geC/H5ApfSQQLgHQRSRORIJwOn0wvx+QVIiI47b+rVPVxb8fjbap6j6omq2oqzv+L/6pqpzzrOxKqugXIEZH+nlWnAyu9GJI3bQZGiUgXz9/N6XTSjvMAbwfQFlS1RkSmALNwev5fVNUVXg7LW8YAVwLLRGSxZ929qjrDeyGZduYm4DXPSdMG4Bovx+MVqvqtiEwHvse5224RnXSoCRtiwhhjfJyvNA0ZY4w5CEsExhjj4ywRGGOMj7NEYIwxPs4SgTHG+DhLBMa0IRE5xUY4Ne2NJQJjjPFxlgiMaYKIXCEi34nIYhF51jNfwQ4R+ZtnfPrPRSTeU3aYiMwXkaUi8p5njBpEpK+IfCYiS0TkexHp4zl8eL3x/l/zPLVqjNdYIjCmAREZCFwKjFHVYUAtcDkQBmSp6mDgC+ABzy4vA3ep6tHAsnrrXwOeUtWhOGPUFHjWDwduxZkbozfO097GeI1PDDFhTDOdDowAFnhO1kOBbTjDVL/pKfMq8K5n/P4oVf3Cs/4/wNsiEgEkqep7AKpaCeA53neqmutZXgykAl+5/qmMOQhLBMY0JsB/VPWeA1aK/K5BuZaOz7Kn3uta7O/QeJk1DRnT2OfARSLSDUBEYkSkF87fy0WeMpcBX6lqGVAiIid61l8JfOGZ/S1XRM73HCNYRLq05Ycw5kjZmYgxDajqShG5D5gtIn5ANXAjziQtIz3btuH0IwBcBTzjqejrj9Z5JfCsiEz1HOPiNvwYxhwxG33UmCMkIjtUNdzbcRjT2qxpyBhjfJxdERhjjI+zKwJjjPFxlgiMMcbHWSIwxhgfZ4nAGGN8nCUCY4zxcf8PLRSLwzcXFfgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(f1s)\n",
    "plt.plot(f1s_eval)\n",
    "plt.title('model f1 micro')\n",
    "plt.ylabel('f1')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'val'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7763846261160714"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max(f1s_eval)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sequence labelling"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Бывают задачи, когда нужно предсказать какой-то класс каждому элементу последовательности. Это называется sequence labelling (разметка последовательности). Под этот паттерн попадает предсказание части речи, определение именованных сущностей, определение опечаток и т.п.\n",
    "\n",
    "Выше мы использовали h (последний стейт), а теперь нужно взять output."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Возьмем данные opencorpora"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 224
    },
    "colab_type": "code",
    "id": "M0V-1lxg50NF",
    "outputId": "90e5c19e-d7d4-40bf-fb51-285d80faea9a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2021-06-04 11:51:01--  http://opencorpora.org/files/export/annot/annot.opcorpora.no_ambig.nonmod.xml.zip\n",
      "Resolving opencorpora.org (opencorpora.org)... 46.4.87.221\n",
      "Connecting to opencorpora.org (opencorpora.org)|46.4.87.221|:80... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 7164759 (6.8M) [application/zip]\n",
      "Saving to: ‘annot.opcorpora.no_ambig.nonmod.xml.zip’\n",
      "\n",
      "annot.opcorpora.no_ 100%[===================>]   6.83M  22.1MB/s    in 0.3s    \n",
      "\n",
      "2021-06-04 11:51:01 (22.1 MB/s) - ‘annot.opcorpora.no_ambig.nonmod.xml.zip’ saved [7164759/7164759]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!wget http://opencorpora.org/files/export/annot/annot.opcorpora.no_ambig.nonmod.xml.zip "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "id": "JHLr7nmv-F3c",
    "outputId": "8ea35612-0630-47d9-b28e-fa77a532c558"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Archive:  annot.opcorpora.no_ambig.nonmod.xml.zip\n",
      "  inflating: annot.opcorpora.no_ambig.nonmod.xml  \n"
     ]
    }
   ],
   "source": [
    "!unzip annot.opcorpora.no_ambig.nonmod.xml.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: lxml in /opt/conda/lib/python3.8/site-packages (4.6.3)\r\n"
     ]
    }
   ],
   "source": [
    "!pip install lxml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "dh8OpwE5-F6h"
   },
   "outputs": [],
   "source": [
    "from lxml import etree\n",
    "from collections import defaultdict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "GCuVu_w9-bC2"
   },
   "outputs": [],
   "source": [
    "open_corpora = etree.fromstring(open('annot.opcorpora.no_ambig.nonmod.xml', 'rb').read())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "KAi-Sq0A9num"
   },
   "source": [
    "Преобразуем в более удобный формат."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "PWp7bYeD-bEg"
   },
   "outputs": [],
   "source": [
    "sents = []\n",
    "labels = []\n",
    "\n",
    "\n",
    "for sentence in open_corpora.xpath('//tokens'):\n",
    "    length = len(sentence.xpath('token'))\n",
    "    ended = False\n",
    "    sent = []\n",
    "    label = []\n",
    "    for i,token in enumerate(sentence.xpath('token')):\n",
    "        word = token.xpath('@text')\n",
    "        gram_info = token.xpath('tfr/v/l/g/@v')\n",
    "        \n",
    "        if (i+1)==length and gram_info[0] == 'PNCT':\n",
    "            gram_info = ['SENT']\n",
    "            ended = True\n",
    "        \n",
    "            \n",
    "        sent.append(word[0])\n",
    "        label.append(gram_info[0])\n",
    "    sents.append(sent)\n",
    "    labels.append(label)\n",
    "\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Пример данных"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('«', 'PNCT'),\n",
       " ('Школа', 'NOUN'),\n",
       " ('злословия', 'NOUN'),\n",
       " ('»', 'PNCT'),\n",
       " ('учит', 'VERB'),\n",
       " ('прикусить', 'INFN'),\n",
       " ('язык', 'NOUN')]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(zip(sents[0], labels[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "# словарь\n",
    "vocab = Counter()\n",
    "\n",
    "for sent in sents:\n",
    "    vocab.update([x.lower() for x in sent])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "54634"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# тут словарь не очень большой и тут нужно предсказывать нужно часть речи для всего\n",
    "# поэтому не будем фильтровать словарь\n",
    "len(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# индексируем слова\n",
    "word2id = {'PAD':0}\n",
    "\n",
    "for word in vocab:\n",
    "    word2id[word] = len(word2id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "id2word = {i:word for word, i in word2id.items()}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "В отличие от простой классификации тут нужно предсказывать последовательность классов. Для обучения нужно закодировать классы точно также как и тексты"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "tags = set()\n",
    "for seq in labels:\n",
    "    tags.update(seq)\n",
    "\n",
    "label2id = {'PAD':0}\n",
    "for tag in tags:\n",
    "    label2id[tag] = len(label2id)\n",
    "\n",
    "    \n",
    "id2label = {i:l for l, i in label2id.items()}\n",
    "tags = [id2label[i] for i in range(len(label2id))]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "И паддить также нужно и тексты и таргеты, чтобы длины у них совпадали"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_LEN = max(len(s) for s in sents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Dataset(torch.utils.data.Dataset):\n",
    "\n",
    "    def __init__(self, word2id, label2id, MAX_LEN, texts, labels):\n",
    "        \n",
    "        # храним сырой датасет\n",
    "        # еще лучше можно сделать, если считывать из файла при генерации\n",
    "        self.texts = [torch.LongTensor([word2id[w.lower()] for w in t][:MAX_LEN]) for t in texts]\n",
    "        self.texts = torch.nn.utils.rnn.pad_sequence(self.texts, batch_first=True)\n",
    "        self.targets = [torch.LongTensor([label2id[l] for l in seq][:MAX_LEN]) for seq in labels]\n",
    "        self.targets = torch.nn.utils.rnn.pad_sequence(self.targets, batch_first=True)\n",
    "        \n",
    "        # мы не считаем словарь прямо тут, т.к. нам нужно шерить словарь между трейном и валидацией\n",
    "        self.word2id = word2id\n",
    "        self.label2id = label2id\n",
    "        self.MAX_LEN = MAX_LEN\n",
    "        self.length = len(texts)\n",
    "    \n",
    "    def __len__(self):\n",
    "        # нужно уметь возвращать длину датасета, в нашем случае это просто сделать\n",
    "        # если не загружать данные целиком, а читать из файла - это может быть сложнее\n",
    "        return self.length\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        ids = self.texts[index]\n",
    "        y = self.targets[index]\n",
    "\n",
    "        return ids, y\n",
    "    \n",
    "    def preprocess(self, text):\n",
    "        tokens = text.lower().split()\n",
    "        tokens = [token.strip(punctuation) for token in tokens]\n",
    "        tokens = [token for token in tokens if token]\n",
    "        return tokens\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 256"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_texts, valid_texts, train_labels, valid_labels = train_test_split(sents, labels, test_size=0.05)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_set = Dataset(word2id, label2id, MAX_LEN, train_texts, train_labels)\n",
    "training_generator = torch.utils.data.DataLoader(training_set, batch_size=BATCH_SIZE, shuffle=True, )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "valid_set = Dataset(word2id, label2id, MAX_LEN, valid_texts, valid_labels)\n",
    "valid_generator = torch.utils.data.DataLoader(valid_set, batch_size=BATCH_SIZE, shuffle=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Для сравнения сначала обучим сеть только с помощью Dense слоев."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import f1_score\n",
    "\n",
    "def train(model, iterator, optimizer, criterion, print_every=1000):\n",
    "    \n",
    "    epoch_loss = []\n",
    "    model.train()  \n",
    "\n",
    "    for i, (texts, ys) in enumerate(iterator):\n",
    "        \n",
    "        optimizer.zero_grad()   \n",
    "        predictions = model(texts.to(device))\n",
    "        loss = criterion(predictions.permute(0, 2, 1), ys.to(device))\n",
    "        \n",
    "        loss.backward()\n",
    "        optimizer.step()     \n",
    "        preds = predictions.detach().to('cpu').numpy().argmax(1).tolist()\n",
    "        y_true = ys.tolist()\n",
    "        \n",
    "        epoch_loss.append(loss.item())\n",
    "        \n",
    "        if not (i+1) % print_every:\n",
    "            print(f'Loss: {np.mean(epoch_loss)}')\n",
    "        \n",
    "    return np.mean(epoch_loss)\n",
    "\n",
    "\n",
    "def evaluate(model, iterator, criterion):\n",
    "    \n",
    "    epoch_loss = []\n",
    "    \n",
    "    model.eval()  \n",
    "    with torch.no_grad():\n",
    "        for texts, ys in iterator:   \n",
    "            \n",
    "            predictions = model(texts.to(device)).squeeze()  \n",
    "            loss = criterion(predictions.permute(0, 2, 1), ys.to(device))        \n",
    "            preds = predictions.detach().to('cpu').numpy().argmax(1).tolist()\n",
    "            y_true = ys.tolist()\n",
    "            \n",
    "\n",
    "            epoch_loss.append(loss.item())\n",
    "            \n",
    "    return np.mean(epoch_loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Не будем делать пакинг, чтобы было попроще"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Clf(nn.Module):\n",
    "    \n",
    "    def __init__(self, output_dim):\n",
    "        \n",
    "        super().__init__()          \n",
    "        self.VOCAB_SIZE = len(word2id)\n",
    "        self.EMB_DIM = 100\n",
    "        self.LSTM_UNITS = 128\n",
    "        \n",
    "        \n",
    "        self.embedding = nn.Embedding(self.VOCAB_SIZE, self.EMB_DIM)\n",
    "        self.rnn = nn.LSTM(self.EMB_DIM, self.LSTM_UNITS, num_layers=1, bidirectional=True,\n",
    "                            batch_first=True) \n",
    "\n",
    "        \n",
    "        self.fc = nn.Linear(self.LSTM_UNITS*2,output_dim)\n",
    "        self.act = nn.LogSoftmax(2) \n",
    "        \n",
    "        \n",
    "    def forward(self, text):\n",
    "\n",
    "        embedded = self.embedding(text)\n",
    "        rnn_output, (h, *_) = self.rnn(embedded)\n",
    "        hidden = self.fc(rnn_output)\n",
    "        outputs=self.act(hidden)\n",
    "\n",
    "        return outputs\n",
    "\n",
    "\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')  \n",
    "model = Clf(len(label2id))\n",
    "optimizer = optim.Adam(model.parameters(), lr=0.001)\n",
    "criterion = nn.NLLLoss() \n",
    "model = model.to(device)\n",
    "criterion = criterion.to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Ф-меру тут расчитать будет сложнее, поэтому пока просто посчитаем лосс, а в конце сделаем общую ф-меру"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "Loss: 0.575894824663798\n",
      "Loss: 0.3530729463944832\n",
      "Loss: 0.27161992482013175\n",
      "1\n",
      "Loss: 0.08987584536274275\n",
      "Loss: 0.08617440412441889\n",
      "Loss: 0.08291676135526763\n",
      "2\n",
      "Loss: 0.06852500326931477\n",
      "Loss: 0.06616999277224143\n",
      "Loss: 0.06410420549412568\n",
      "3\n",
      "Loss: 0.05456961207091808\n",
      "Loss: 0.05416192971169949\n",
      "Loss: 0.05310431623624431\n",
      "4\n",
      "Loss: 0.04675999681154887\n",
      "Loss: 0.04601671596368154\n",
      "Loss: 0.044871056121256615\n",
      "5\n",
      "Loss: 0.03987998552620411\n",
      "Loss: 0.03890239285926024\n",
      "Loss: 0.03783602499299579\n",
      "6\n",
      "Loss: 0.033493749673167865\n",
      "Loss: 0.03269342494507631\n",
      "Loss: 0.03208171038164033\n",
      "7\n",
      "Loss: 0.02867095023393631\n",
      "Loss: 0.02804863496373097\n",
      "Loss: 0.027490093807379403\n",
      "8\n",
      "Loss: 0.0247644929215312\n",
      "Loss: 0.02418655725196004\n",
      "Loss: 0.023728503100574016\n",
      "9\n",
      "Loss: 0.021308195715149244\n",
      "Loss: 0.02092552436515689\n",
      "Loss: 0.020674644514090484\n"
     ]
    }
   ],
   "source": [
    "losses = []\n",
    "losses_eval = []\n",
    "\n",
    "for i in range(10):\n",
    "    print(i)\n",
    "    losses.append(train(model, training_generator, optimizer, criterion, 30))\n",
    "    losses_eval.append(evaluate(model, valid_generator, criterion))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAA0ZUlEQVR4nO3deXxU9b3/8dcnk30lCSEJCZuIQFiUEhCl7oIoCN661qXW2lrvT6td7L3a5bbXelvv7XJbK63i0trWil60LQqKqLhVRYIiO7LIkhAgLCH7/vn9cU7CJAxhApmcZObzfDzmkZmzzHwSJe98l/M9oqoYY4wxHUV5XYAxxpjeyQLCGGNMQBYQxhhjArKAMMYYE5AFhDHGmIAsIIwxxgRkAWFMNxCRP4rIA0Eeu11ELj7Z9zEm1CwgjDHGBGQBYYwxJiALCBMx3K6d74rIahGpFpEnRCRbRF4WkUoReU1E0v2Ony0i60SkXETeFJHRfvsmiMhH7nnPAvEdPmuWiKxyz31PRMafYM1fE5EtInJQRBaKyEB3u4jI/4rIPhGpEJE1IjLW3XeZiKx3aysRkXtO6AdmIp4FhIk0VwLTgNOAy4GXge8BWTj/Hu4CEJHTgGeAb7r7FgMvikisiMQCfwf+DGQA/+e+L+65E4Anga8DmcCjwEIRietKoSJyIfAz4BogF9gBzHd3TwfOdb+PNPeYA+6+J4Cvq2oKMBZ4oyufa0wrCwgTaX6rqntVtQR4B1iuqh+rah3wN2CCe9y1wCJVXaqqjcAvgATgbGAKEAP8WlUbVXUBsMLvM24DHlXV5ararKpPAfXueV1xA/Ckqn6kqvXAfcBZIjIUaARSgFGAqOoGVS11z2sECkQkVVUPqepHXfxcYwALCBN59vo9rw3wOtl9PhDnL3YAVLUF2AXkuftKtP1Klzv8ng8BvuN2L5WLSDkwyD2vKzrWUIXTSshT1TeAh4G5wD4RmSciqe6hVwKXATtE5C0ROauLn2sMYAFhzLHsxvlFDzh9/ji/5EuAUiDP3dZqsN/zXcB/qWo/v0eiqj5zkjUk4XRZlQCo6kOqOhEowOlq+q67fYWqzgEG4HSFPdfFzzUGsIAw5lieA2aKyEUiEgN8B6eb6D3gfaAJuEtEYkTkC8Bkv3MfA24XkTPdweQkEZkpIildrOEZ4BYROcMdv/gpTpfYdhGZ5L5/DFAN1AEt7hjJDSKS5naNVQAtJ/FzMBHMAsKYAFR1E3Aj8FtgP86A9uWq2qCqDcAXgC8DB3HGK17wO7cI+BpOF9AhYIt7bFdreA34IfA8TqtlOHCduzsVJ4gO4XRDHQB+7u67CdguIhXA7ThjGcZ0mdgNg4wxxgRiLQhjjDEBWUAYY4wJyALCGGNMQBYQxhhjAor2uoDu0r9/fx06dKjXZRhjTJ+ycuXK/aqaFWhf2ATE0KFDKSoq8roMY4zpU0Rkx7H2WReTMcaYgCwgjDHGBGQBYYwxJqCwGYMIpLGxkeLiYurq6rwuJeTi4+PJz88nJibG61KMMWEirAOiuLiYlJQUhg4dSvuFN8OLqnLgwAGKi4sZNmyY1+UYY8JEWHcx1dXVkZmZGdbhACAiZGZmRkRLyRjTc8I6IICwD4dWkfJ9GmN6TtgHxHG1NEPFbmiq97oSY4zpVSwgtBmqy6CiJCRvX15ezu9+97sun3fZZZdRXl7e/QUZY0yQLCB8sZCcDXWHob6y29/+WAHR1NTU6XmLFy+mX79+3V6PMcYEywICICkLomKcVkQ330Dp3nvvZevWrZxxxhlMmjSJc845h9mzZ1NQUADAFVdcwcSJExkzZgzz5s1rO2/o0KHs37+f7du3M3r0aL72ta8xZswYpk+fTm1tbbfWaIwxgYT1NFd///niOtbvrjj2AS1N0FQH0fucsAhCwcBUfnT5mE6PefDBB1m7di2rVq3izTffZObMmaxdu7ZtOuqTTz5JRkYGtbW1TJo0iSuvvJLMzMx277F582aeeeYZHnvsMa655hqef/55brzxxqBqNMaYE2UtiFZR0SBR0NQAhO42rJMnT253rcJDDz3E6aefzpQpU9i1axebN28+6pxhw4ZxxhlnADBx4kS2b98esvqMMaZVxLQgjveXPgAN1bD/U0jOgdTckNSRlJTU9vzNN9/ktdde4/333ycxMZHzzz8/4LUMcXFxbc99Pp91MRljeoS1IPzFJkF8OlTtc1sSJy8lJYXKysCD34cPHyY9PZ3ExEQ2btzIBx980C2faYwx3SFiWhBBS82FunKo3A3pQ0/67TIzM5k6dSpjx44lISGB7Ozstn0zZszgkUceYfTo0YwcOZIpU6ac9OcZY0x3Ee3mWTvt3lxkBvAbwAc8rqoPdth/O3AH0AxUAbep6noRGQpsADa5h36gqrd39lmFhYXa8YZBGzZsYPTo0V0vvGI3VO2F/qc5rYo+4oS/X2NMxBKRlapaGGhfyFoQIuID5gLTgGJghYgsVNX1fof9VVUfcY+fDfwKmOHu26qqZ4Sqvk4lZ0PNAThcAv1HgC1jYYyJQKEcg5gMbFHVbaraAMwH5vgfoKr+806TCOX0oa6I8kHKQGishtpDXldjjDGeCGVA5AG7/F4Xu9vaEZE7RGQr8D/AXX67honIxyLyloicE+gDROQ2ESkSkaKysrLurB0SMyA6ASpLoaWle9/bGGP6AM9nManqXFUdDvw78AN3cykwWFUnAN8G/ioiqQHOnaeqhapamJWV1b2FiUBaHjQ3QPW+7n1vY4zpA0IZECXAIL/X+e62Y5kPXAGgqvWqesB9vhLYCpwWmjI7EZcC8WnOgHVzY49/vDHGeCmUAbECGCEiw0QkFrgOWOh/gIiM8Hs5E9jsbs9yB7kRkVOAEcC2ENZ6bKkDnfWZKnd78vHGGOOVkM1iUtUmEbkTWIIzzfVJVV0nIvcDRaq6ELhTRC4GGoFDwM3u6ecC94tII9AC3K6qB0NVa6ei453F/Kr3QWIWxCaG9OOSk5OpqqoK6WcYY0wwQnqhnKouBhZ32PYffs/vPsZ5zwPPh7K2LknJhtqDzmqvmafatFdjTESwK6mDERUNKTlwuNi5b0RCv6BPvffeexk0aBB33HEHAD/+8Y+Jjo5m2bJlHDp0iMbGRh544AHmzJlznHcyxpieFTkB8fK9sGfNSbyBQmON8zQmERDIGQeXPtjpWddeey3f/OY32wLiueeeY8mSJdx1112kpqayf/9+pkyZwuzZs+2+0saYXiVyAuKkCfjioKnWmdHkiw3qrAkTJrBv3z52795NWVkZ6enp5OTk8K1vfYu3336bqKgoSkpK2Lt3Lzk5OSH+HowxJniRExDH+Us/aAe2QEMNDBgNvuBuLHT11VezYMEC9uzZw7XXXsvTTz9NWVkZK1euJCYmhqFDhwZc5tsYY7zk+YVyfU5qHmgzVO4J+pRrr72W+fPns2DBAq6++moOHz7MgAEDiImJYdmyZezYsSOEBRtjzImJnBZEd4lJgMT+ULMfkvo7r49jzJgxVFZWkpeXR25uLjfccAOXX34548aNo7CwkFGjRvVA4cYY0zUWECciJddZxK912msQ1qw5MkDev39/3n///YDH2TUQxpjewrqYToTPnfZaXwl1Fcc/3hhj+iALiBOV1N+Z1VRR4izFYYwxYSbsAyJkd8yTKGedpqY6ZzzCY6G8M6AxJjKFdUDEx8dz4MCB0P3yjE+D2GSoKIWWptB8RhBUlQMHDhAfH+9ZDcaY8BPWg9T5+fkUFxfT7TcT8tfc4Ex5La3u0hIc3S0+Pp78/HzPPt8YE37COiBiYmIYNmxY6D/oH4/BJ/PhjuWQOTz0n2eMMT0grLuYesyFP4ToOFj6H8c/1hhj+ggLiO6Qkg2f/xZsfAk+e9vraowxpltYQHSXs+6AtEHwyvegpdnraowx5qRZQHSXmASY9p+wdw2setrraowx5qRZQHSnMV+AQWfC6z9xrrI2xpg+zAKiO4nAJT9z7l/97v96XY0xxpyUkAaEiMwQkU0iskVE7g2w/3YRWSMiq0TkXREp8Nt3n3veJhG5JJR1dqv8iTDuGnjvYSjf6XU1xhhzwkIWECLiA+YClwIFwBf9A8D1V1Udp6pnAP8D/Mo9twC4DhgDzAB+575f33Dxj5ylOF77sdeVGGPMCQtlC2IysEVVt6lqAzAfmON/gKr6L4WaBLSuiTEHmK+q9ar6GbDFfb++IS0fzv4GrH0edi73uhpjjDkhoQyIPGCX3+tid1s7InKHiGzFaUHc1cVzbxORIhEpCulyGidi6t2QnANL7oOWFq+rMcaYLvN8kFpV56rqcODfgR908dx5qlqoqoVZWVmhKfBExSU7XU0lK2HtAq+rMcaYLgtlQJQAg/xe57vbjmU+cMUJnts7jb8Ocs9wxiIaaryuxhhjuiSUAbECGCEiw0QkFmfQeaH/ASIywu/lTGCz+3whcJ2IxInIMGAE8GEIaw2NqCiY8TPnpkLvP+x1NcYY0yUhW81VVZtE5E5gCeADnlTVdSJyP1CkqguBO0XkYqAROATc7J67TkSeA9YDTcAdqto3168YcjaMnu1cFzHhJkjN9boiY4wJioTLncgKCwu1qKjI6zICO/gZzJ0M466GK37ndTXGGNNGRFaqamGgfZ4PUkeEjGFw5u3OGk27P/a6GmOMCYoFRE859x5I7O+s9homrTZjTHizgOgp8Wlw4fdh53uwYeHxjzfGGI9ZQPSkCV+CAQXOneea6r2uxhhjOmUB0ZN80XDJf8Gh7bD8Ea+rMcaYTllA9LThF8KIS+DtX0BVL1sexBhj/FhAeGH6A9BYA8v+y+tKjDHmmCwgvJB1Gkz6Knz0FOxd53U1xhgTkAWEV877d4hLhSXft2mvxpheyQLCK4kZcP69sG0ZbH7V62qMMeYoFhBemvRVyDzVaUU0N3pdjTHGtGMB4SVfjDNgfWAzFD3pdTXGGNOOBYTXTpsBw86DZT+FmoNeV2OMMW0sILwmApf8FOor4K3/8boaY4xpYwHRG+SMhc99CVY8Bp8u8boaY4wBLCB6jwv/AwaMhr9eA6/fD81NXldkjIlwER8QTc0tvLt5P3sO13lbSFIm3LrUaUm880v48xVQtc/bmowxES3iA2JPRR03PrGcFz4u9roUiEmA2b+FK34PxUXwyDmw/Z9eV2WMiVARHxD56YlMGNyPRatLvS7liDOuh6+9DrFJ8NTl8O6v7WprY0yPC2lAiMgMEdkkIltE5N4A+78tIutFZLWIvC4iQ/z2NYvIKvcR0jvszByXy7rdFXy2vzqUH9M12WPgtjdh9OXw2o9g/vVQW+51VcaYCBKygBARHzAXuBQoAL4oIgUdDvsYKFTV8cACwH+eZ62qnuE+ZoeqToDLxuUCsHhNL2pFAMSnwtV/hBn/7SzH8ei5sHuV11UZYyJEKFsQk4EtqrpNVRuA+cAc/wNUdZmq1rgvPwDyQ1jPMQ3sl8DEIem81Ju6mVqJwJTb4ZaXoaUJnpgORX+wLidjTMiFMiDygF1+r4vdbcdyK/Cy3+t4ESkSkQ9E5IoQ1NfOzHG5bCitYGtZVag/6sQMmgxffweGfh5e+ib87XZo6EVdYsaYsNMrBqlF5EagEPi53+YhqloIXA/8WkSGBzjvNjdEisrKTu7ubJeNy0WE3jVY3VFSJtywAC74Pqx+Fh67CMo+9boqY0yYCmVAlACD/F7nu9vaEZGLge8Ds1W1vnW7qpa4X7cBbwITOp6rqvNUtVBVC7Oysk6q2Jy0eCYNyejdAQEQFQXn/Rvc9AJU74PHLoA1C7yuyhgThkIZECuAESIyTERigeuAdrORRGQC8ChOOOzz254uInHu8/7AVGB9CGsFYOb4XDbtrWTz3spQf9TJG36h0+WUPQaevxUWfxea6o9/njHGBClkAaGqTcCdwBJgA/Ccqq4TkftFpHVW0s+BZOD/OkxnHQ0UicgnwDLgQVUNeUBcOjbH6WbqbbOZjiUtD768CM66Ez6cB3+4FMp3el2VMSZMiIbJbJjCwkItKio66fe59tH3OVjdwNJvn9cNVfWg9QvhH3dAlA++8BiMmOZ1RcaYPkBEVrrjvUfpFYPUvcms8bls3lfFp32hm8lfwWznwrrUfHj6Knj9J9DS7HVVxpg+zAKigxljc4kSeOmT3V6X0nWZw+GrS2HCTfDOL2zBP2PMSbGA6CArJY4pp2Ty0ppS+mT3W0wCzHkY5syFXR86C/7teM/rqowxfZAFRAAzx+eyrayajXv6WDeTvwk3wldfh9hE+OMs+OdDdvW1MaZLLCACmDEmh6jeftFcMHLGOuMSo2bC0h/Cszfagn/GmKBZQASQmRzH2cP7s6ivdjP5i0+Da/4EMx6ET1+BeefZgn/GmKBYQBzDzPG5fLa/mvWlFV6XcvJEYMq/Ogv+NTc6C/6t/KN1ORljOmUBcQyXjMnBFyW9c4XXEzVoMnz9bRg6FV68G/7+r7bgnzHmmCwgjiEjKZapp/Zn0eow6Gbyl9TfWfDv/Pvgk/nOgn/7N3tdlTGmF7KA6MSscbnsPFjD2pIw6GbyF+WD8++FG593Fvybdz6sfcHrqowxvYwFRCemj8kmOkp4aU0fvGguGKde5Cz4N6AAFtwCi/8Nmhq8rsoY00tYQHSiX2Isnx8Rht1M/tLy4JbFMOUO+PBRd8G/Xcc/zxgT9iwgjmPmuFyKD9Wyuviw16WEji8GZvzUmQ5btgke+TysfApaWryuzBjjIQuI45hekEOMT/rOEuAno2AOfP0tp8vpxbvgDzNg7zqvqzLGeMQC4jjSEmM4d0RWeHcz+csc7nQ5zfmdM7vpkXPg1R/adFhjIpAFRBBmjs+lpLyWj3eVe11KzxCBCTfAN1bCGdfDew/B3DNh42KvKzPG9CALiCBcXJBNrC+q76/N1FWJGc7KsF9ZAnEpMP+L8Mz1NohtTISwgAhCanwM556WxeI1pbS0REA3U0eDpzhXYE+7H7Ytg7mTndVhmxu9rswYE0IWEEGaNT6X0sN1fLzrkNeleMMXA1PvhjuWw7DznNVhHz0Pdi73ujJjTIhYQATpotEDiI2OCq+1mU5Ev8Fw/Xy47q9QdxienA4L74Kag15XZozpZkEFhIjcLSKp4nhCRD4SkelBnDdDRDaJyBYRuTfA/m+LyHoRWS0ir4vIEL99N4vIZvdxc9e+re6XEh/D+ZHczdTRqJlOa+Lsb8DHf4GHC2HVM7ZCrDFhJNgWxFdUtQKYDqQDNwEPdnaCiPiAucClQAHwRREp6HDYx0Chqo4HFgD/456bAfwIOBOYDPxIRNKDrDVkZp0+kL0V9RTtiNBupo7ikmH6A874RMZw+Pvtzt3ryjZ5XZkxphsEGxDifr0M+LOqrvPbdiyTgS2quk1VG4D5wBz/A1R1marWuC8/APLd55cAS1X1oKoeApYCM4KsNWQuGjWAuOgoFq0O07WZTlTOWGem0+W/gb1r4fdT4fX7oaHm+OcaY3qtYANipYi8ihMQS0QkBTjeOgx5gP98yGJ327HcCrzclXNF5DYRKRKRorKysuOUc/KS4qK5cNQAFq/dQ7N1M7UXFQUTvwx3FsG4q+CdX8LvpsDmpV5XZow5QcEGxK3AvcAk9y/+GOCW7ipCRG4ECoGfd+U8VZ2nqoWqWpiVldVd5XRq5vhcyirrWbHdBmUDSs6Cf3kEbn4JouPg6avguS9BhbW6jOlrgg2Is4BNqlru/jL/AXC81etKgEF+r/Pdbe2IyMXA94HZqlrflXO9cOGoAcTHROBFc1017By4/Z9w4Q/h0yXw8CT44PfQ3OR1ZcaYIAUbEL8HakTkdOA7wFbgT8c5ZwUwQkSGiUgscB2w0P8AEZkAPIoTDvv8di0BpotIujs4Pd3d5rnE2GguGpXNy2tLaWq21U47FR0L594D/+8D52K7V+6Fxy6A4pVeV2aMCUKwAdGkzkp1c4CHVXUukNLZCaraBNyJ84t9A/Ccqq4TkftFZLZ72M+BZOD/RGSViCx0zz0I/AQnZFYA97vbeoVZ43PZX9XAh5/1mpJ6t4xhzm1Or34Kqsvg8Ytg0XegttzryowxnZBgVigVkbeAV4CvAOcA+4BPVHVcaMsLXmFhoRYVFfXIZ9U2NDPxgaVcMSGPn/5Lr/kR9A11FbDsp87NiRL7w4yfwdgrnQUCjTE9TkRWqmphoH3BtiCuBepxrofYgzMm0KUB5XCSEOvjotHZvLJ2j3UzdVV8Klz6IHxtGaTlw/O3wp+vgANbva7MGNNBUAHhhsLTQJqIzALqVPV4YxBhbea4XA5WN/DBNutmOiEDz4CvvgaX/QJKPnKmxC77GTTWeV2ZMcYV7FIb1wAfAlcD1wDLReSqUBbW250/MoukWB+L1tj0zRMW5YPJX3OunSiYA289CL8/C7a+4XVlxhiC72L6Ps41EDer6pdwrpL+YejK6v3iY3xcXOB0MzVaN9PJScmGKx+Hm/4OCPz5X2DBrVC5x+vKjIlowQZEVIdpqAe6cG7YmjV+IIdqGnlv6wGvSwkPwy+Af30Pzr8PNiyEXxU4azu9PxcObvO6OmMiTrC/5F8RkSUi8mUR+TKwCIj4+0+eM6I/KXHRtjZTd4qJh/Pvda6d+Py3oOYALPkePDTBue3pa/8Ju1ZAi7XajAm1oKa5AojIlcBU9+U7qvq3kFV1Anpymqu/bz+7itc37mPF9y8mNjriG1WhcfAz+PQV2LQYtv8TtBmSBsBpl8DIy+CU8yE20esqjemTOpvmGnRA9HZeBcTrG/Zy61NF/OGWSVwwckCPf37EqT0Em19zwmLLa1BfAdEJTvfUyEvhtBmQbP8djAlWZwERfZwTK4FACSKAqmpqN9TXp31+RH9S4qNZtLrUAqInJKTD+KudR1MD7PgnbHrZCYxNiwGB/EKnZTHyMsgaaRfhGXOCrAXRDb7z3Ce8un4PK38wzbqZvKLq3IuiNSx2f+xsTx/mhsWlMPgs8HX6N5ExEeeEWxAmOLNOz+X5j4p5Z3MZF43O9rqcyCQCOeOcx3n/5iwv/ukrsHExrHgMPpgL8f3ccYtLYfhFzlXdxphjsoDoBlOH9yctIYZFq0stIHqL1IFQ+BXnUV/lXHy36WUnNFY/C1ExzpLkra2LtPzjv6cxEcYCohvERkdxyZhsXl6zh7rGZuJjfF6XZPzFJUPBbOfR0gy7ljvdUBsXw+J7nEfO+CNhkXu6jVsYg41BdJu3Pi3j5ic/5LEvFTKtwFoRfcb+ze4A98uw8wNAITXPCYqRl8LQc5w74xkTpmwMogecPTyTfokxLFq92wKiL+k/AvrfDVPvhur9zt3vNi2GVX+FFY9DbLIzbjH2Sjj1YgsLE1EsILpJjC+KGWNyePGT3dbN1Fcl9YcJNziPxjr47G3Y+BJseBHWPg/xaTD6chh7FQw711ls0JgwZnMyu9HM8blUNzTz1qdlXpdiTlZMPJw2HWY/BPd8Cjc874xRrPuHc/+KX46Cxd+FncudKbbGhCFrQXSjs07JJCMplpdWl3LJmByvyzHdxRcDIy52HrNqYfOrsGYBrHwKPpwHaYNh7Bdg3FWQPdYGuE3YsIDoRtG+KGaMzeHvH5dQ29BMQqx1QYSdmATn3hUFc5zbp25c5HQ/vfdb+Oevof9IJyjGXgmZw72u1piTEtIuJhGZISKbRGSLiNwbYP+5IvKRiDR1vAGRiDSLyCr3sTCUdXanWeNyqWlo5s1N+45/sOnb4lPhjC/CjQvgns0w81fOOMay/4Lffg4ePc8JjsMlXldqzAkJ2TRXEfEBnwLTgGJgBfBFVV3vd8xQIBW4B1ioqgv89lWpanKwn+f1NNdWTc0tTPnZ65x5SiZzr/+c1+UYLxwugXUvON1QpasAgSFnO62KgisgKdPjAo05orNprqFsQUwGtqjqNlVtAOYDc/wPUNXtqroaCJvF/Vu7md7YsI+ahiavyzFeSMuDs78BX38LvvERXPA9Zwrtom/DL0bAX66EVc84XVTG9GKhDIg8YJff62J3W7DiRaRIRD4QkSsCHSAit7nHFJWV9Z6ZQzPHDaS2sZllG3tPTcYjmcOdtaHuWA63v+sER9mn8PfbnbB49iZY/w9orPW6UmOO0psHqYeoaomInAK8ISJrVHWr/wGqOg+YB04XkxdFBjJ5WAZZKXG8tHo3M8fnel2O6Q38FxO8+MdQvMLpglr3N+f2qrEpMGqmM8B9yvnOzCljPBbKgCgBBvm9zne3BUVVS9yv20TkTWACsLXTk3oJX5Rw2dgc5q/YRXV9E0lxvTmHTY8TgUGTncclP4Xt78DaBbD+RVg9HxIznVlSY69yliiPssuVjDdC+X/eCmCEiAwTkVjgOiCo2Ugiki4ice7z/ji3Ol3f+Vm9y8zxA6lvauH1jTabyXTCF+3cDW/OXPjuZrjur04L4pP58MfL4NdjYcn3oeQjuyDP9LiQBYSqNgF3AkuADcBzqrpORO4XkdkAIjJJRIqBq4FHRWSde/pooEhEPgGWAQ/6z37qCwqHpDMgJY5Fq3d7XYrpK6LjnG6mq550ps1e+YSzyuzyR+GxC+D3U6HoSWf5cmN6gK3mGkI/XriOv364k49+OI1k62YyJ6rmoDOQXfQE7FkDcalw+hdh0lch6zSvqzN9nFfTXCPerPG5NDS18PqGvV6XYvqyxAwovAW+/g585VU4bYbTkpg7CZ6a7Swm2GxTqk33s4AIoc8NTic3LZ4XPyn1uhQTDkRg8Jlw5WPw7Q1w4Q/hwFZ49kb4zenw9s+hysa8TPexgAihqCjhsnG5vP1pGRV1jV6XY8JJchacew/c/Qlc+zT0PxXeeAB+VQALbnVufhQm3cfGOxYQITZzfC4NzS28tt66mUwI+KJh9Cz40j/gziJnXGLzUnjyEnjkHFj5R2io9rpK00dZQITYhEH9yOuXwKLV1s1kQqz/CLj0QfjOBpj1a0Dhxbvhl6Phlftg/xavKzR9jAVEiIkIl43L4e3NZRyutW4m0wNik5xB7dvfhVtece5j8eFj8PBE+NMVzhLlLc1eV2n6AAuIHjBz/EAam5Wl1s1kepIIDDnLua7iW+vggh9A2SaYf70zqP3OL51FBI05BguIHnB6fhr56Ql20ZzxTko2nPdd+OYauObPkDEMXr8ffjUaXrgNdq2wQW1zFLt6qweICDPH5/LEO59RXtNAv8RYr0sykcoXDQWznUfZJljxuLP0+Opnnau2J3/NWQMqNtHrSk0vYC2IHjJr3ECaWpRX11k3k+klskbCZT+H72x07obX0gQLv+G0KpZ837nGwkQ0C4geMjYvlcEZiby0xmYzmV4mLhkm3Qr/+h58ebGzeODyR5zbpv7lStj0sg1qRyjrYuohrd1M897exqHqBtKTrJvJ9DIiMHSq86jcAyufgpV/gGeug36DofArMOFLdsvUCGKL9fWgtSWHmfXbd3nwC+O4bvJgr8sx5viaG51psSsed+5bERXjDHD3G+KERvoQ53nr14R0J2hMn9HZYn3WguhBYwamMjQzkUVrSi0gTN/gi4ExVziPfRtg9XNwYAuU73DuildX3v742JQjYXFUgAyGuBQPvglzoiwgepCIMGv8QH7/1lYOVNWTmRzndUnGBG/AaLj4R+231R2GQzugfKcTGod2uF8/g21vQmOHZT4SMgIEyFDna9ogiInvqe/GBMECoofNHJ/Lw8u28Mq6Pdxw5hCvyzHm5MSnQe5459GRKtQccENjuxMirQGyZw1sWgzNDe3PSc45utXR+jw135mma3qM/bR72KicFE7JSmLR6lILCBPeRCCpv/PIn3j0/pYWqNrj1+rwa4ns/MC5T7e2+L2fD1LzjgRHSi6k5kLKQPdrLiRlQZSv577HMGcB0cNEhFnjnFZEWWU9WSnWzWQiVFQUpA50HkPOOnp/cyMcLj66+6p8J2x9A6r2tg8QcEIkJcd95DrvnZLjFyLu6/jUnvke+zgLCA/MHD+Qh95wuplummKtCGMC8rkzpjKGBd7f3ATVZVC5GypKodJ9tD4/sAU+ewfqDx99bmyyEyApOW6I5B7dIknOdmqIYBYQHjgtO5lTBySzaPVuCwhjTpQv2vlFnpoLeZ0c11DtXNdRsdv52jFQdrzvfG3puNqyOF1Wrd1XgVokqQMhvl/YTu0NaUCIyAzgN4APeFxVH+yw/1zg18B44DpVXeC372bgB+7LB1T1qVDW2pOc2Uy5/Ob1zeyrrGNAis3cMCZkYpMgc7jzOJaWFqg9eOwQOVwCxUVQE2D129hkZwZWv0Htv7Y+T85xutP6oJAFhIj4gLnANKAYWCEiC1V1vd9hO4EvA/d0ODcD+BFQCCiw0j33UKjq7Wkzx+Xy69c28/KaPdx89lCvyzEmskVFHRlQDzQjq1VTvRsgbogcLnHGSQ7vcsZGildAbYdfU1ExkJbnBsZgNzzy/YIkH6J751hkKFsQk4EtqroNQETmA3OAtoBQ1e3uvg4jTVwCLFXVg+7+pcAM4JkQ1tujRmSnMDI7hUWrSy0gjOkrouOcWVTpnXQN11e5gbHL+er/fOsyp0WC/woW4ox3HKsFkjbIs0H1UAZEHrDL73UxcOZJnHtUL6OI3AbcBjB4cN+7Mnnm+Fz+97VP2XO4jpw062YyJizEJTsXFQ4YHXh/UwNUlLjhUeyGx07na+kq2PjS0deHxKdB2mA3MPL9wsPdlpQVknGQPj1IrarzgHngrMXkcTlddtm4XH619FNeXlvKLVOPMVPDGBNeomM7n53V0gLV+wK3QA7tgO3vQn1F+3MGToDb3uz+Urv9HY8oAQb5vc53twV77vkdzn2zW6rqRU4dkMyoHKebyQLCGAM44yGt13IMmhT4mNry9i2QEN3gKZQBsQIYISLDcH7hXwdcH+S5S4Cfiki6+3o6cF/3l+i9y08fyM+XbKL0cC25aQlel2OM6QsS+jmPnHEh/ZiQzb1S1SbgTpxf9huA51R1nYjcLyKzAURkkogUA1cDj4rIOvfcg8BPcEJmBXB/64B1uLlsXC4Ai1bbjYSMMb2L3Q+iF5j50DvERkfxt/831etSjDERprP7QfTNqzfCzMzxuXy8s5ziQzVel2KMMW0sIHqBmW4308tr9nhciTHGHGEB0QsMyUxiXF4aL62xcQhjTO9hAdFLzBqfyye7yvnvVzayalc5LS3hMTZkjOm7+vSFcuHkmsJBvLtlP/Pe3sbv39xKdmocF4/OZlpBNmcNzyQu2m6CYozpWTaLqZcpr2ngjY37WLp+L299WkZNQzPJcdGcNzKL6QXZnD9yAGkJkb1GvTGm+3Q2i8kCohera2zmva37Wbp+L0vX72N/VT3RUcKUUzKZVuC0Lgb2s4vrjDEnzgIiDLS0KB/vKmfp+r28un4P28qqARibl8q00TlMH5PNqJwUJExvXGKMCQ0LiDC0tazKCYt1e/h4VzmqkJ+e0NaymDw0g2ifzUEwxnTOAiLMlVXW8/qGvby6fi/vbtlPQ1ML/RJjuHDkAKYVZHPuaVkkxdl8BGPM0SwgIkh1fRPvbC7j1fV7eWPjPsprGomNjuLzp/ZnWkE2F40eYLc4Nca06Swg7M/KMJMUF82MsbnMGJtLU3MLK7Yfahu3eGPjPkRgwqB+TCtwxi2GZyV7XbIxppeyFkSEUFU27qlsC4u1Jc4NR07JSmJaQTbTC7KZMCidqCgb5DYmklgXkznK7vJaXtuwl1fX7eWDbQdoalH6J8dx8egBXDBqAOPy0shNi7dZUcaEOQsI06nDtY28ucm5OO/NTWVU1TcB0C8xhlE5KYzOTaUgN5XRuamMyE62q7qNCSMWECZo9U3NrCk+zIbSCtaXVrKhtIJNeyqpbWwGIDpKGJ6VzOhcJzhaH1kpcR5Xbow5ETZIbYIWF+2jcGgGhUMz2rY1tyg7DlSzvrSCDaUVbCitZPlnB/n7qt1tx2SlxLlhkdLW2jilf5Jdi2FMH2YBYY7LFyWckpXMKVnJzBo/sG37oeoGNuxxAmNDaQXrd1fwh60HaGhuASA2OorTspPbAqP1YWtJGdM3WBeT6VaNzS1sLatqa2m0BseB6oa2Y/L6JbTroirITWVwRqLNoDLGA551MYnIDOA3gA94XFUf7LA/DvgTMBE4AFyrqttFZCiwAdjkHvqBqt4eylpN94jxRTEqJ5VROan8ywRnm6pSVlnvdlFVuuFRwbJNZTS7971IivUxMqf9uMbInBSS7QpwYzwTsn99IuID5gLTgGJghYgsVNX1fofdChxS1VNF5Drgv4Fr3X1bVfWMUNVneo6IMCA1ngGp8Zw/ckDb9rrGZjbvrWJ96WE2lFayvrSChZ/s5unlO9uOyUyKZVBGIkMyExmSkeg+T2JIZiJZyXHW6jAmhEL559lkYIuqbgMQkfnAHMA/IOYAP3afLwAeFpt4HzHiY3yMy09jXH5a2zZVpaS8lg2llWzZV8XOg9XsPFjDyh2HePGT3fjfaC8uOorBbngMynACZEhmEoMyEhmUkWDTcY05SaEMiDxgl9/rYuDMYx2jqk0ichjIdPcNE5GPgQrgB6r6TscPEJHbgNsABg8e3L3VG0+ICPnpieSnJzKtILvdvoamFnaX17LjYA07DzjBseNADTsP1vDe1gPUNDT7vQ/kpsYfaX24wTHEfZ2WEGMXARpzHL21g7cUGKyqB0RkIvB3ERmjqhX+B6nqPGAeOIPUHtRpelBsdBRD+ycxtH8SkNVun6qyv6qhrcWx40ANO93wWLapjLLK4nbHp8RHu91WSW0hMjjDeQzsl4DPuq6MCWlAlACD/F7nu9sCHVMsItFAGnBAnalV9QCqulJEtgKnATZNyQQkImSlxJGVEsfEIRlH7a9paGLXwVp2uC2P1hBZX1rBq+v30Nh85O+LGJ+Q1y+BwZlJDMlIJD89gYH9EshLTyCvX4KNfZiIEcqAWAGMEJFhOEFwHXB9h2MWAjcD7wNXAW+oqopIFnBQVZtF5BRgBLAthLWaMJcYG83InBRG5qQcta+5RSk9XOsEx4EapwvLfb5q5yEq6praHR/jE3LS4hmY5gTGwLZHPHn9Esjtl2Czr0xYCNn/xe6Ywp3AEpxprk+q6joRuR8oUtWFwBPAn0VkC3AQJ0QAzgXuF5FGoAW4XVUPhqpWE9l8UUfGPc4efvT+irpGSsvr2F1eS0l5LbvbHnUs/+wgeyrq2qbrtkpLiHFaHf3i/QLkyOsBKfHWjWV6PbtQzpiT1NTcQllVPbvLayk+5ARHa4i0BkrHVogvSshJjXdbIPF+AXKkNZISb1ecm9CztZiMCaFoXxS5aQnkpiUwcUjgYyrrGik9XHdUC6SkvJaiHYfYs7qUpg6tkJT46HaBkZ0Sz4BUZ5xlQEo8WSlxZCbF2npXJmQsIIzpASnxMaTEx3Ba9tFjIOCMg5RV1ncIkFpK3NbIRzsPUV7TeNR5Is7FhP2T4xiQGk9WcpwTIu2+OmFi4yKmq+z/GGN6AV+UM/CdkxbPxCHpAY+pa2xmf1U9+yrrKXMfR57XUVZZz+a9lZRV1h/VGgFIjPU5M706hkdyHFmtr1PiyEyOs/ERA1hAGNNnxMf42gbTO9PSohyubWwLj31uePgHyqY9lbxbuf+osRGAKIGMJCcsnO6suLYpxANS4klPiiEjKZaMxFj6JcYSG21dXOHKAsKYMBMVJaQnxZKeFBtwWq+/usZmJzyq6tlX4Xwtq6hzvrqB8mknrRKA5LhoJzQSnc9MT3QeGUkxHV7Hkp4UQ3piLDE2btInWEAYE8HiY3zu2lXHb5WU1zZSVlnPweoGDtU0cLC6gfKaBg5WN7a9PljdwNayKg5VN7bdujaQlLjothDLSHRCIz3JDZHEWNITY9q97pcYY6HiAQsIY8xxRUWJ062UFBv0OfVNzZTXHAmPQ9WNHKxp4JAbMIeqGzhY08j+qgY276viUHUD1X7raXWUEh/dLkBSE2JIjY8hNSHa/RrodTSpCRYuJ8oCwhgTEnHRPrJTfWSnxgd9Tl2jEyptrZPWIHFbKa1hU1ZVz7b91VTUNlJR13TUhYodJcT4AgaHBUznLCCMMb1GfIyPnDQfOWnBh4qqUtPQTEVdIxW1Te7XxiOv/Z/XOc/3VzV0S8CkxEeTHBdDcpyPpLhokt1HUlw0yfF+z91HX5sdZgFhjOnTRIQk9xdxbtrxj+/IP2AO13YMFSdAOoZMWVU9W8uqqahrpLq+qd1ij52Jj4kKGCjJ8dFHB0ycj+S4GJLifEeOiT2yvydmj1lAGGMiWvuASTih96hvaqa6vpnq+iYq65qobmiiqr6Jqromquvd5/Wtz5v9njexp6KOqrIjr+saW4L6zNjoqLZAOX1QP377xQknVHtnLCCMMeYkxUX7iIv2dWkQ/1iamluorm+mqsEJGP8waX1eXd9EZdvzZnK70CXXFRYQxhjTi0T7okhLjCIt0fvFGiNzaN4YY8xxWUAYY4wJyALCGGNMQBYQxhhjArKAMMYYE5AFhDHGmIAsIIwxxgRkAWGMMSYgUQ1uDZHeTkTKgB0n8Rb9gf3dVE5fZz+L9uzn0Z79PI4Ih5/FEFXNCrQjbALiZIlIkaoWel1Hb2A/i/bs59Ge/TyOCPefhXUxGWOMCcgCwhhjTEAWEEfM87qAXsR+Fu3Zz6M9+3kcEdY/CxuDMMYYE5C1IIwxxgRkAWGMMSagiA8IEZkhIptEZIuI3Ot1PV4SkUEiskxE1ovIOhG52+uavCYiPhH5WERe8roWr4lIPxFZICIbRWSDiJzldU1eEpFvuf9O1orIMyISmtu6eSiiA0JEfMBc4FKgAPiiiBR4W5WnmoDvqGoBMAW4I8J/HgB3Axu8LqKX+A3wiqqOAk4ngn8uIpIH3AUUqupYwAdc521V3S+iAwKYDGxR1W2q2gDMB+Z4XJNnVLVUVT9yn1fi/ALI87Yq74hIPjATeNzrWrwmImnAucATAKraoKrlnhblvWggQUSigURgt8f1dLtID4g8YJff62Ii+BeiPxEZCkwAlntcipd+Dfwb0OJxHb3BMKAM+IPb5fa4iCR5XZRXVLUE+AWwEygFDqvqq95W1f0iPSBMACKSDDwPfFNVK7yuxwsiMgvYp6orva6ll4gGPgf8XlUnANVAxI7ZiUg6Tm/DMGAgkCQiN3pbVfeL9IAoAQb5vc53t0UsEYnBCYenVfUFr+vx0FRgtohsx+l6vFBE/uJtSZ4qBopVtbVFuQAnMCLVxcBnqlqmqo3AC8DZHtfU7SI9IFYAI0RkmIjE4gwyLfS4Js+IiOD0MW9Q1V95XY+XVPU+Vc1X1aE4/1+8oaph9xdisFR1D7BLREa6my4C1ntYktd2AlNEJNH9d3MRYThoH+11AV5S1SYRuRNYgjML4UlVXedxWV6aCtwErBGRVe6276nqYu9KMr3IN4Cn3T+mtgG3eFyPZ1R1uYgsAD7Cmf33MWG47IYttWGMMSagSO9iMsYYcwwWEMYYYwKygDDGGBOQBYQxxpiALCCMMcYEZAFhTC8gIufbirGmt7GAMMYYE5AFhDFdICI3isiHIrJKRB517xdRJSL/694b4HURyXKPPUNEPhCR1SLyN3f9HkTkVBF5TUQ+EZGPRGS4+/bJfvdbeNq9QtcYz1hAGBMkERkNXAtMVdUzgGbgBiAJKFLVMcBbwI/cU/4E/LuqjgfW+G1/GpirqqfjrN9T6m6fAHwT594kp+Bc2W6MZyJ6qQ1juugiYCKwwv3jPgHYh7Mc+LPuMX8BXnDvn9BPVd9ytz8F/J+IpAB5qvo3AFWtA3Df70NVLXZfrwKGAu+G/Lsy5hgsIIwJngBPqep97TaK/LDDcSe6fk293/Nm7N+n8Zh1MRkTvNeBq0RkAICIZIjIEJx/R1e5x1wPvKuqh4FDInKOu/0m4C33Tn3FInKF+x5xIpLYk9+EMcGyv1CMCZKqrheRHwCvikgU0AjcgXPznMnuvn044xQANwOPuAHgv/rpTcCjInK/+x5X9+C3YUzQbDVXY06SiFSparLXdRjT3ayLyRhjTEDWgjDGGBOQtSCMMcYEZAFhjDEmIAsIY4wxAVlAGGOMCcgCwhhjTED/HxGLdb6GDGhoAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(losses)\n",
    "plt.plot(losses_eval)\n",
    "plt.title('model loss')\n",
    "plt.ylabel('loss')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'val'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Лучше конечно еще оценивать правильность всей последовательности, но пока сделаем только поэлементную оценку."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "# предсказываем весь валидационный сет\n",
    "pred = model(valid_set.texts.to(device)).argmax(2).detach().cpu()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "         PAD       1.00      1.00      1.00     66659\n",
      "        INFN       0.59      0.33      0.43       222\n",
      "        TIME       0.00      0.00      0.00         2\n",
      "        PRTF       0.00      0.00      0.00        74\n",
      "        ADVB       0.74      0.75      0.74       280\n",
      "        INTJ       0.00      0.00      0.00        26\n",
      "        NUMB       0.79      0.78      0.79       318\n",
      "        PRTS       0.00      0.00      0.00        83\n",
      "        UNKN       0.31      0.15      0.20       288\n",
      "        COMP       0.00      0.00      0.00        23\n",
      "        LATN       0.70      0.45      0.55       207\n",
      "        SENT       0.99      1.00      1.00      1111\n",
      "        Prnt       0.00      0.00      0.00         0\n",
      "        PNCT       0.99      1.00      1.00      1499\n",
      "        VERB       0.63      0.73      0.67       939\n",
      "        SYMB       0.00      0.00      0.00        16\n",
      "        NPRO       0.92      0.94      0.93       282\n",
      "        CONJ       0.97      0.97      0.97       561\n",
      "        PREP       0.98      0.99      0.99      1216\n",
      "        ROMN       0.00      0.00      0.00        14\n",
      "        PRED       0.88      0.79      0.84        29\n",
      "        DATE       0.00      0.00      0.00         1\n",
      "        PRCL       0.94      0.92      0.93       253\n",
      "        ADJS       1.00      0.03      0.07        59\n",
      "        NUMR       0.00      0.00      0.00        24\n",
      "        ADJF       0.70      0.75      0.73      1243\n",
      "        HANI       0.00      0.00      0.00         9\n",
      "        NOUN       0.82      0.91      0.86      3887\n",
      "        GREK       0.00      0.00      0.00         4\n",
      "        GRND       0.00      0.00      0.00        32\n",
      "\n",
      "   micro avg       0.97      0.97      0.97     79361\n",
      "   macro avg       0.47      0.42      0.42     79361\n",
      "weighted avg       0.97      0.97      0.97     79361\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# сплющиваем последовательности классов в 1 последовательность и \n",
    "# считаем метрики как будто это обычная классификация\n",
    "print(classification_report(valid_set.targets.reshape(-1), pred.reshape(-1), labels=list(range(len(tags))),\n",
    "                                                                             target_names=tags,\n",
    "                                                                             zero_division=0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Предсказание/Inference"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "После оценки часто хочется посмотреть как будет работать модель на каких-то отдельных примерах. Для этого нужно воспрозвести весь пайплайн и прогонять через него нужный текст."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "def tokenize(text, word2id):\n",
    "    # токенизирует и переводит в индексы\n",
    "    tokens = re.findall('\\w+|[^\\w\\s]+', text)\n",
    "    \n",
    "    # в тф тут мы должны были еще думать а паддинге\n",
    "    # но в торче на единичных примерах падить не нужно\n",
    "    ids = [word2id.get(token.lower(), 0) for token in tokens]\n",
    "    return tokens, ids\n",
    "\n",
    "def pred2tags(pred, id2label):\n",
    "    # декодирует индексы в части речи\n",
    "    pred = pred.argmax(2)[0].numpy()\n",
    "\n",
    "    labels = [id2label[l] for l in pred]\n",
    "    return labels\n",
    "\n",
    "def label_seq(text, word2id, id2label, max_len, model):\n",
    "    tokens, ids = tokenize(text, word2id)\n",
    "    pred = model(torch.LongTensor([ids]))\n",
    "    labels = pred2tags(pred, id2label)\n",
    "    \n",
    "    return list(zip(tokens, labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Clf(\n",
       "  (embedding): Embedding(54635, 100)\n",
       "  (rnn): LSTM(100, 128, batch_first=True, bidirectional=True)\n",
       "  (fc): Linear(in_features=256, out_features=30, bias=True)\n",
       "  (act): LogSoftmax(dim=2)\n",
       ")"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.to('cpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Какой', 'ADJF'),\n",
       " ('-', 'PNCT'),\n",
       " ('то', 'CONJ'),\n",
       " ('новый', 'ADJF'),\n",
       " ('текст', 'NOUN'),\n",
       " ('для', 'PREP'),\n",
       " ('проверки', 'NOUN'),\n",
       " ('работы', 'NOUN'),\n",
       " ('.', 'PNCT')]"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label_seq('Какой-то новый текст для проверки работы.', word2id, id2label, MAX_LEN, model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Выглядит неплохо!"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}