mrr-testing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "51954ba6",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/data/home/admin/t5-experiments-from-yura/venv/lib/python3.8/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import math\n",
    "from tqdm import tqdm\n",
    "import numpy as np\n",
    "from typing import Dict\n",
    "from collections import defaultdict\n",
    "from torch.utils.data import DataLoader\n",
    "import torch\n",
    "from pymongo import MongoClient\n",
    "from torch.utils.data import Dataset\n",
    "from transformers import AutoConfig, AutoTokenizer, HfArgumentParser, T5ForConditionalGeneration, T5Config, PretrainedConfig\n",
    "import os\n",
    "from transformers import (\n",
    "    LogitsProcessorList,\n",
    "    MinLengthLogitsProcessor,\n",
    "    BeamSearchScorer,\n",
    ")\n",
    "from typing import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "beb68b2e",
   "metadata": {},
   "outputs": [],
   "source": [
    "entities = []\n",
    "with open(\"wd5m_aliases_entities_v3.txt\", 'r') as f:\n",
    "    for line in f:\n",
    "        entities.append(str(line.split('\\t')[1][:-1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "28fb46ae",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(['human',\n",
       "  'United States of America',\n",
       "  'taxon',\n",
       "  'species',\n",
       "  'United Kingdom',\n",
       "  'English',\n",
       "  'association football',\n",
       "  'politician',\n",
       "  'association football player',\n",
       "  'UTC+01:00'],\n",
       " 4818679)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "entities[:10], len(entities)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "28de2b61",
   "metadata": {},
   "outputs": [],
   "source": [
    "class KGLMDataset(Dataset):\n",
    "    def __init__(self, port, db, collection):\n",
    "        self.client = MongoClient('localhost', port)\n",
    "        self.db_name = db\n",
    "        self.collection_name = collection\n",
    "        self.collection = self.client[db][collection]\n",
    "        self.tokenizer = AutoTokenizer.from_pretrained(\"t5-base\")\n",
    "        self.tokenizer.add_tokens(['[SEP]'], special_tokens=True)\n",
    "        self.length = self.client[self.db_name].command(\"collstats\", self.collection_name)['count']\n",
    "\n",
    "    def  __getitem__(self, idx):\n",
    "        item = {}\n",
    "        doc = self.collection.find_one({'_id': str(idx)})\n",
    "        item[\"input\"] = doc['verbalization']\n",
    "        item[\"outputs\"] = doc['target']\n",
    "        return item\n",
    "        \n",
    "    def __len__(self):\n",
    "        return self.length\n",
    "    \n",
    "    def _collate_eval(self, batch):\n",
    "        encode_plus_kwargs = {'truncation': True, 'padding': 'longest', 'pad_to_multiple_of': 1}\n",
    "\n",
    "        \n",
    "        inputs = [b['input'] for b in batch]\n",
    "        inputs_tokenized = self.tokenizer.batch_encode_plus(list(inputs), max_length=512, return_tensors='pt',\n",
    "                                                   **encode_plus_kwargs)\n",
    "        \n",
    "        target_text = [b[\"outputs\"] for b in batch]\n",
    "\n",
    "        return inputs_tokenized.input_ids, inputs_tokenized.attention_mask, target_text, inputs\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "dc501354",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/data/home/admin/t5-experiments-from-yura/venv/lib/python3.8/site-packages/transformers/models/t5/tokenization_t5_fast.py:156: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5.\n",
      "For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`.\n",
      "- Be aware that you SHOULD NOT rely on t5-base automatically truncating your input to 512 when padding/encoding.\n",
      "- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding.\n",
      "- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "dataset = KGLMDataset(27017, 'KGLM', 'test')\n",
    "data_loader = DataLoader(\n",
    "    dataset,\n",
    "    batch_size=1,\n",
    "    shuffle=False,\n",
    "    num_workers=1,\n",
    "    collate_fn=dataset._collate_eval\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "669dc385",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = 'lr5e-05_constant_with_warmup_adamw_wd1e-03_512-512_bs64_iters4000000/run_1/'\n",
    "\n",
    "model_cpt = os.path.join(path, 'model_best.pth')\n",
    "config_path = os.path.join(path, 'config.json')\n",
    "\n",
    "model_cfg = AutoConfig.from_pretrained('t5-small')\n",
    "model = T5ForConditionalGeneration(config=model_cfg)\n",
    "\n",
    "cpt = torch.load(model_cpt, map_location='cpu')\n",
    "model.load_state_dict(cpt['model_state_dict'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b483efff",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Args:\n",
    "    def __init__(self, entity_strings=entities, batch_size=2, chunk_size=100, num_workers=2, device='cuda'):\n",
    "        self.batch_size = batch_size\n",
    "        self.num_workers = num_workers\n",
    "        self.chunk_size = chunk_size\n",
    "        self.device = device\n",
    "        self.entity_strings = entity_strings\n",
    "\n",
    "args = Args()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "c347dac2",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Evaluator:\n",
    "    def __init__(self, dataset: KGLMDataset, model, args):\n",
    "        self.device = args.device\n",
    "        self.dataset = dataset\n",
    "        self.model = model.to(self.device)\n",
    "        self.num_workers = args.num_workers\n",
    "        self.batch_size = args.batch_size\n",
    "        self.chunk_size = args.chunk_size\n",
    "        self.entity_strings = args.entity_strings\n",
    "        self.ent2id = {ent: i for i, ent in enumerate(self.entity_strings)}\n",
    "        self.data_loader = DataLoader(\n",
    "            dataset,\n",
    "            batch_size=self.batch_size,\n",
    "            shuffle=False,\n",
    "            num_workers=self.num_workers,\n",
    "            collate_fn=dataset._collate_eval,\n",
    "        )\n",
    "#         self.filter_dicts = dict()\n",
    "#         self.filter_dicts[\"train\"] = self.create_filter_dict(\"train\")\n",
    "#         self.filter_dicts[\"valid\"] = self.create_filter_dict(\"valid\")\n",
    "#         self.filter_dicts[\"test\"] = self.create_filter_dict(\"test\")\n",
    "\n",
    "#     def create_filter_dict(self, split: str) -> Dict[str, int]:\n",
    "#         data = self.dataset.split(split)\n",
    "#         filter_dict = defaultdict(list)\n",
    "#         for input, output in zip(data[\"inputs\"], data[\"outputs\"]):\n",
    "#             filter_dict[input].append(self.dataset.entity_string_to_id[output])\n",
    "#         return filter_dict\n",
    "\n",
    "    @torch.no_grad()\n",
    "    def eval(self):\n",
    "        self.model.eval()\n",
    "        loader = tqdm(self.data_loader, total=len(self.data_loader), unit=\"batch\")\n",
    "        ranks = {\n",
    "            \"unfiltered\": list(),\n",
    "            \"filtered\": list(),\n",
    "        }\n",
    "        for steps, batch in enumerate(loader):\n",
    "            ranks_in_batch = {\n",
    "                \"unfiltered\": list(),\n",
    "                \"filtered\": list()\n",
    "            }\n",
    "            \n",
    "            input_ids, attention_mask, label_strings, input_strings = batch\n",
    "            input_ids = input_ids.to(self.device)\n",
    "            attention_mask = attention_mask.to(self.device)\n",
    "            \n",
    "            # labels = labels.to(self.device)\n",
    "#             input_ids_repeated = torch.repeat_interleave(\n",
    "#                 input_ids, len(self.dataset.entity_strings), dim=0\n",
    "#             )\n",
    "#             attention_mask_repeated = torch.repeat_interleave(\n",
    "#                 attention_mask, len(self.dataset.entity_strings), dim=0\n",
    "#             )\n",
    "#             tokenized_entities = self.dataset.tokenized_entities.input_ids.to(\n",
    "#                 self.device\n",
    "#             )\n",
    "            # todo: for filtering we need to use only the filtered entities per triple here\n",
    "#             all_entities_repeated = tokenized_entities.repeat([self.batch_size, 1])\n",
    "            summed_logit_chunks = []\n",
    "            # process chunk by chunk\n",
    "            for chunk_number in tqdm(range(\n",
    "                math.ceil(len(self.entity_strings) / self.chunk_size)\n",
    "            )):\n",
    "                chunk_start = self.chunk_size * chunk_number\n",
    "                chunk_end = min(\n",
    "                    self.chunk_size * (chunk_number + 1), len(self.entity_strings)\n",
    "                )\n",
    "                current_chunk_size = chunk_end - chunk_start\n",
    "                \n",
    "                input_ids_repeated = torch.repeat_interleave(\n",
    "                    input_ids, current_chunk_size, dim=0\n",
    "                )\n",
    "                \n",
    "                attention_mask_repeated = torch.repeat_interleave(\n",
    "                    attention_mask, current_chunk_size, dim=0\n",
    "                )\n",
    "                \n",
    "                entities_repeated_chunk = \\\n",
    "                    self.dataset.tokenizer.batch_encode_plus(self.entity_strings[chunk_start:chunk_end], padding='max_length', \n",
    "                    truncation=True, max_length=32, return_tensors=\"pt\").input_ids.to(self.device)\n",
    "                \n",
    "                entities_repeated_chunk = entities_repeated_chunk.repeat([self.batch_size, 1])\n",
    "                \n",
    "                outputs_chunk = self.model(\n",
    "                    input_ids=input_ids_repeated,\n",
    "                    attention_mask=attention_mask_repeated,\n",
    "                    labels=entities_repeated_chunk,\n",
    "                )\n",
    "\n",
    "                \n",
    "                logits_chunk = outputs_chunk.logits\n",
    "                soft_logits_chunk = torch.log_softmax(logits_chunk, dim=2)\n",
    "\n",
    "                coordinates = entities_repeated_chunk.view(current_chunk_size*self.batch_size, -1, 1)\n",
    "\n",
    "                # set padded logits to zero\n",
    "                padded_mask = (coordinates == 0).squeeze()\n",
    "                soft_logits_chunk[padded_mask] = 0\n",
    "                needed_soft_logits_chunk = torch.gather(\n",
    "                    soft_logits_chunk,\n",
    "                    2,\n",
    "                    coordinates\n",
    "#                 )\n",
    "                ).view(self.batch_size, current_chunk_size, -1)\n",
    "                \n",
    "                summed_logits = torch.sum(needed_soft_logits_chunk, dim=2)\n",
    "                summed_logit_chunks.append(summed_logits)\n",
    "                \n",
    "            summed_logits = torch.cat(summed_logit_chunks, dim=1)\n",
    "            \n",
    "            for summed_logits_per_triple, input_string, label in zip(\n",
    "                summed_logits, input_strings, label_strings\n",
    "            ):\n",
    "                # todo: currently we are calculating best rank on equality\n",
    "                #  change to mean\n",
    "                arg_sorted = torch.argsort(summed_logits_per_triple, descending=True)\n",
    "                print(arg_sorted)\n",
    "                \n",
    "                entity_id = self.ent2id[label]\n",
    "                print(entity_id)\n",
    "                rank = (\n",
    "                    (arg_sorted == entity_id)\n",
    "                    .nonzero(as_tuple=True)[0]\n",
    "                    .item()\n",
    "                )\n",
    "                print(rank)\n",
    "                ranks_in_batch[\"unfiltered\"].append(rank)\n",
    "\n",
    "                # now filter\n",
    "#                 true_score = summed_logits_per_triple[entity_id].clone()\n",
    "#                 for filter_dict in self.filter_dicts.values():\n",
    "#                     summed_logits_per_triple[filter_dict[input_string]] = -float(\"inf\")\n",
    "#                 summed_logits_per_triple[entity_id] = true_score\n",
    "#                 arg_sorted = torch.argsort(summed_logits_per_triple, descending=True)\n",
    "#                 rank = (\n",
    "#                     (arg_sorted == entity_id)\n",
    "#                         .nonzero(as_tuple=True)[0]\n",
    "#                         .item()\n",
    "#                 )\n",
    "#                 print(rank)\n",
    "#                 ranks_in_batch[\"filtered\"].append(rank)\n",
    "#             ranks[\"filtered\"].extend(ranks_in_batch[\"filtered\"])\n",
    "            ranks[\"unfiltered\"].extend(ranks_in_batch[\"unfiltered\"])\n",
    "        for setting, list_of_ranks in ranks.items():\n",
    "            ranks[setting] = np.array(list_of_ranks, dtype=np.float32) + 1\n",
    "        # ranks = np.array(ranks, dtype=np.float32)\n",
    "        # # add 1 to have best rank 1 not 0\n",
    "        # ranks += 1\n",
    "        print(\"MR\", ranks[\"unfiltered\"].mean())\n",
    "#         print(\"MR-filtered\", ranks[\"filtered\"].mean())\n",
    "        print(\"MRR\", np.power(ranks[\"unfiltered\"], -1).mean())\n",
    "        print(\"MRR-filtered\", np.power(ranks[\"filt /ered\"], -1).mean())\n",
    "        print(\"Hits@1\", (ranks[\"unfiltered\"] == 1).sum() / len(self.dataset))\n",
    "#         print(\"Hits@1-filtered\", (ranks[\"filtered\"] == 1).sum() / len(self.dataset))\n",
    "        print(\"Hits@10\", (ranks[\"unfiltered\"] <= 10).sum() / len(self.dataset))\n",
    "#         print(\"Hits@10-filtered\", (ranks[\"filtered\"] <= 10).sum() / len(self.dataset))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "becb5fa3",
   "metadata": {},
   "outputs": [],
   "source": [
    "evaluator = Evaluator(dataset, model, args)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "c5675578",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|                                               | 0/2567 [00:00<?, ?batch/s]/data/home/admin/t5-experiments-from-yura/venv/lib/python3.8/site-packages/pymongo/topology.py:177: UserWarning: MongoClient opened before fork. Create MongoClient only after forking. See PyMongo's documentation for details: https://pymongo.readthedocs.io/en/stable/faq.html#is-pymongo-fork-safe\n",
      "  warnings.warn(\n",
      "/data/home/admin/t5-experiments-from-yura/venv/lib/python3.8/site-packages/pymongo/topology.py:177: UserWarning: MongoClient opened before fork. Create MongoClient only after forking. See PyMongo's documentation for details: https://pymongo.readthedocs.io/en/stable/faq.html#is-pymongo-fork-safe\n",
      "  warnings.warn(\n",
      "\n",
      "  0%|                                                 | 0/48187 [00:00<?, ?it/s]\u001b[A\n",
      "  0%|                                       | 1/48187 [00:00<3:33:28,  3.76it/s]\u001b[A\n",
      "  0%|                                       | 2/48187 [00:00<2:44:54,  4.87it/s]\u001b[A\n",
      "  0%|                                       | 3/48187 [00:00<2:40:05,  5.02it/s]\u001b[A\n",
      "  0%|                                       | 4/48187 [00:00<2:40:23,  5.01it/s]\u001b[A\n",
      "  0%|                                       | 5/48187 [00:01<2:40:44,  5.00it/s]\u001b[A\n",
      "  0%|                                       | 6/48187 [00:01<2:41:26,  4.97it/s]\u001b[A\n",
      "  0%|                                       | 7/48187 [00:01<2:41:45,  4.96it/s]\u001b[A\n",
      "  0%|                                       | 8/48187 [00:01<2:43:36,  4.91it/s]\u001b[A\n",
      "  0%|                                       | 9/48187 [00:01<2:41:28,  4.97it/s]\u001b[A\n",
      "  0%|                                      | 10/48187 [00:02<2:41:47,  4.96it/s]\u001b[A\n",
      "  0%|                                      | 11/48187 [00:02<2:41:27,  4.97it/s]\u001b[A\n",
      "  0%|                                      | 12/48187 [00:02<2:41:09,  4.98it/s]\u001b[A\n",
      "  0%|                                      | 13/48187 [00:02<2:41:38,  4.97it/s]\u001b[A\n",
      "  0%|                                      | 14/48187 [00:02<2:44:11,  4.89it/s]\u001b[A\n",
      "  0%|                                      | 15/48187 [00:03<2:39:56,  5.02it/s]\u001b[A\n",
      "  0%|                                      | 16/48187 [00:03<2:40:16,  5.01it/s]\u001b[A\n",
      "  0%|                                      | 17/48187 [00:03<2:40:00,  5.02it/s]\u001b[A\n",
      "  0%|                                      | 18/48187 [00:03<2:40:11,  5.01it/s]\u001b[A\n",
      "  0%|                                      | 19/48187 [00:03<2:40:23,  5.01it/s]\u001b[A\n",
      "  0%|                                      | 20/48187 [00:04<2:40:15,  5.01it/s]\u001b[A\n",
      "  0%|                                      | 21/48187 [00:04<2:43:00,  4.92it/s]\u001b[A\n",
      "  0%|                                      | 22/48187 [00:04<2:40:37,  5.00it/s]\u001b[A\n",
      "  0%|                                      | 23/48187 [00:04<2:41:21,  4.98it/s]\u001b[A\n",
      "  0%|                                      | 24/48187 [00:04<2:41:36,  4.97it/s]\u001b[A\n",
      "  0%|                                      | 25/48187 [00:05<2:41:45,  4.96it/s]\u001b[A\n",
      "  0%|                                      | 26/48187 [00:05<2:45:38,  4.85it/s]\u001b[A\n",
      "  0%|                                      | 27/48187 [00:05<2:42:38,  4.94it/s]\u001b[A\n",
      "  0%|                                      | 28/48187 [00:05<2:40:52,  4.99it/s]\u001b[A\n",
      "  0%|                                      | 29/48187 [00:05<2:40:21,  5.01it/s]\u001b[A\n",
      "  0%|                                      | 30/48187 [00:06<2:42:28,  4.94it/s]\u001b[A\n",
      "  0%|                                      | 31/48187 [00:06<2:45:03,  4.86it/s]\u001b[A\n",
      "  0%|                                      | 32/48187 [00:06<2:42:40,  4.93it/s]\u001b[A\n",
      "  0%|                                      | 33/48187 [00:06<2:41:28,  4.97it/s]\u001b[A\n",
      "  0%|                                      | 34/48187 [00:06<2:40:56,  4.99it/s]\u001b[A\n",
      "  0%|                                      | 35/48187 [00:07<2:41:02,  4.98it/s]\u001b[A\n",
      "  0%|                                      | 36/48187 [00:07<2:41:42,  4.96it/s]\u001b[A\n",
      "  0%|                                      | 37/48187 [00:07<2:41:42,  4.96it/s]\u001b[A\n",
      "  0%|                                      | 38/48187 [00:07<2:41:38,  4.96it/s]\u001b[A\n",
      "  0%|                                      | 39/48187 [00:07<2:41:37,  4.96it/s]\u001b[A\n",
      "  0%|                                      | 40/48187 [00:08<2:42:29,  4.94it/s]\u001b[A\n",
      "  0%|                                      | 41/48187 [00:08<2:46:00,  4.83it/s]\u001b[A\n",
      "  0%|                                      | 42/48187 [00:08<2:43:18,  4.91it/s]\u001b[A\n",
      "  0%|                                      | 43/48187 [00:08<2:42:09,  4.95it/s]\u001b[A\n",
      "  0%|                                      | 44/48187 [00:08<2:41:34,  4.97it/s]\u001b[A\n",
      "  0%|                                      | 45/48187 [00:09<2:41:49,  4.96it/s]\u001b[A\n",
      "  0%|                                      | 46/48187 [00:09<2:42:00,  4.95it/s]\u001b[A\n",
      "  0%|                                      | 47/48187 [00:09<2:43:51,  4.90it/s]\u001b[A\n",
      "  0%|                                      | 48/48187 [00:09<2:40:50,  4.99it/s]\u001b[A\n",
      "  0%|                                      | 49/48187 [00:09<2:42:37,  4.93it/s]\u001b[A\n",
      "  0%|                                      | 50/48187 [00:10<2:41:59,  4.95it/s]\u001b[A\n",
      "  0%|                                      | 51/48187 [00:10<2:41:52,  4.96it/s]\u001b[A\n",
      "  0%|                                      | 52/48187 [00:10<2:43:57,  4.89it/s]\u001b[A\n",
      "  0%|                                      | 53/48187 [00:10<2:41:03,  4.98it/s]\u001b[A\n",
      "  0%|                                      | 54/48187 [00:10<2:40:55,  4.98it/s]\u001b[A\n",
      "  0%|                                      | 55/48187 [00:11<2:41:23,  4.97it/s]\u001b[A\n",
      "  0%|                                      | 56/48187 [00:11<2:41:34,  4.96it/s]\u001b[A\n",
      "  0%|                                      | 57/48187 [00:11<2:41:37,  4.96it/s]\u001b[A\n",
      "  0%|                                      | 58/48187 [00:11<2:43:21,  4.91it/s]\u001b[A\n",
      "  0%|                                      | 59/48187 [00:11<2:40:53,  4.99it/s]\u001b[A\n",
      "  0%|                                      | 60/48187 [00:12<2:41:49,  4.96it/s]\u001b[A\n",
      "  0%|                                      | 61/48187 [00:12<2:41:55,  4.95it/s]\u001b[A\n",
      "  0%|                                      | 62/48187 [00:12<2:41:45,  4.96it/s]\u001b[A\n",
      "  0%|                                      | 63/48187 [00:12<2:42:49,  4.93it/s]\u001b[A\n",
      "  0%|                                      | 64/48187 [00:12<2:41:35,  4.96it/s]\u001b[A\n",
      "  0%|                                      | 65/48187 [00:13<2:42:44,  4.93it/s]\u001b[A\n",
      "  0%|                                      | 66/48187 [00:13<2:42:26,  4.94it/s]\u001b[A\n",
      "  0%|                                      | 67/48187 [00:13<2:42:28,  4.94it/s]\u001b[A\n",
      "  0%|                                      | 68/48187 [00:13<2:42:13,  4.94it/s]\u001b[A\n",
      "  0%|                                      | 69/48187 [00:13<2:41:51,  4.95it/s]\u001b[A\n",
      "  0%|                                      | 70/48187 [00:14<2:42:38,  4.93it/s]\u001b[A\n",
      "  0%|                                      | 71/48187 [00:14<2:43:19,  4.91it/s]\u001b[A\n",
      "  0%|                                      | 72/48187 [00:14<2:42:34,  4.93it/s]\u001b[A\n",
      "  0%|                                      | 73/48187 [00:14<2:41:50,  4.96it/s]\u001b[A\n",
      "  0%|                                      | 74/48187 [00:14<2:41:16,  4.97it/s]\u001b[A\n",
      "  0%|                                      | 75/48187 [00:15<2:42:31,  4.93it/s]\u001b[A\n",
      "  0%|                                      | 76/48187 [00:15<2:42:34,  4.93it/s]\u001b[A\n",
      "  0%|                                      | 77/48187 [00:15<2:42:26,  4.94it/s]\u001b[A\n",
      "  0%|                                      | 78/48187 [00:15<2:42:00,  4.95it/s]\u001b[A\n",
      "  0%|                                      | 79/48187 [00:15<2:41:40,  4.96it/s]\u001b[A\n",
      "  0%|                                      | 80/48187 [00:16<2:42:59,  4.92it/s]\u001b[A\n",
      "  0%|                                      | 81/48187 [00:16<2:41:54,  4.95it/s]\u001b[A\n",
      "  0%|                                      | 82/48187 [00:16<2:41:59,  4.95it/s]\u001b[A\n",
      "  0%|                                      | 83/48187 [00:16<2:41:26,  4.97it/s]\u001b[A\n",
      "  0%|                                      | 84/48187 [00:16<2:41:28,  4.97it/s]\u001b[A\n",
      "  0%|                                      | 85/48187 [00:17<2:41:46,  4.96it/s]\u001b[A\n",
      "  0%|                                      | 86/48187 [00:17<2:41:57,  4.95it/s]\u001b[A\n",
      "  0%|                                      | 87/48187 [00:17<2:42:07,  4.94it/s]\u001b[A\n",
      "  0%|                                      | 88/48187 [00:17<2:43:17,  4.91it/s]\u001b[A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|                                      | 89/48187 [00:17<2:41:35,  4.96it/s]\u001b[A\n",
      "  0%|                                      | 90/48187 [00:18<2:42:34,  4.93it/s]\u001b[A\n",
      "  0%|                                      | 91/48187 [00:18<2:42:37,  4.93it/s]\u001b[A\n",
      "  0%|                                      | 92/48187 [00:18<2:42:46,  4.92it/s]\u001b[A\n",
      "  0%|                                      | 93/48187 [00:18<2:42:58,  4.92it/s]\u001b[A\n",
      "  0%|                                      | 94/48187 [00:19<2:41:54,  4.95it/s]\u001b[A\n",
      "  0%|                                      | 95/48187 [00:19<2:42:09,  4.94it/s]\u001b[A\n",
      "  0%|                                      | 96/48187 [00:19<2:41:58,  4.95it/s]\u001b[A\n",
      "  0%|                                      | 97/48187 [00:19<2:42:22,  4.94it/s]\u001b[A\n",
      "  0%|                                      | 98/48187 [00:19<2:42:54,  4.92it/s]\u001b[A\n",
      "  0%|                                      | 99/48187 [00:20<2:41:41,  4.96it/s]\u001b[A\n",
      "  0%|                                     | 100/48187 [00:20<2:41:42,  4.96it/s]\u001b[A\n",
      "  0%|                                     | 101/48187 [00:20<2:41:40,  4.96it/s]\u001b[A\n",
      "  0%|                                     | 102/48187 [00:20<2:41:22,  4.97it/s]\u001b[A\n",
      "  0%|                                     | 103/48187 [00:20<2:42:10,  4.94it/s]\u001b[A\n",
      "  0%|                                     | 104/48187 [00:21<2:42:04,  4.94it/s]\u001b[A\n",
      "  0%|                                     | 105/48187 [00:21<2:41:29,  4.96it/s]\u001b[A\n",
      "  0%|                                     | 106/48187 [00:21<2:43:05,  4.91it/s]\u001b[A\n",
      "  0%|                                     | 107/48187 [00:21<2:41:43,  4.95it/s]\u001b[A\n",
      "  0%|                                     | 108/48187 [00:21<2:42:34,  4.93it/s]\u001b[A\n",
      "  0%|                                     | 109/48187 [00:22<2:41:55,  4.95it/s]\u001b[A\n",
      "  0%|                                     | 110/48187 [00:22<2:42:25,  4.93it/s]\u001b[A\n",
      "  0%|                                     | 111/48187 [00:22<2:45:47,  4.83it/s]\u001b[A\n",
      "  0%|                                     | 112/48187 [00:22<2:41:30,  4.96it/s]\u001b[A\n",
      "  0%|                                     | 113/48187 [00:22<2:42:04,  4.94it/s]\u001b[A\n",
      "  0%|                                     | 114/48187 [00:23<2:44:22,  4.87it/s]\u001b[A\n",
      "  0%|                                     | 115/48187 [00:23<2:42:30,  4.93it/s]\u001b[A\n",
      "  0%|                                     | 116/48187 [00:23<2:43:22,  4.90it/s]\u001b[A\n",
      "  0%|                                     | 117/48187 [00:23<2:43:25,  4.90it/s]\u001b[A\n",
      "  0%|                                     | 118/48187 [00:23<2:42:56,  4.92it/s]\u001b[A\n",
      "  0%|                                     | 119/48187 [00:24<2:42:19,  4.94it/s]\u001b[A\n",
      "  0%|                                     | 120/48187 [00:24<2:44:20,  4.87it/s]\u001b[A\n",
      "  0%|                                     | 121/48187 [00:24<2:43:05,  4.91it/s]\u001b[A\n",
      "  0%|                                     | 122/48187 [00:24<2:42:37,  4.93it/s]\u001b[A\n",
      "  0%|                                     | 123/48187 [00:24<2:43:24,  4.90it/s]\u001b[A\n",
      "  0%|                                     | 124/48187 [00:25<2:42:49,  4.92it/s]\u001b[A\n",
      "  0%|                                     | 125/48187 [00:25<2:43:47,  4.89it/s]\u001b[A\n",
      "  0%|                                     | 126/48187 [00:25<2:43:05,  4.91it/s]\u001b[A\n",
      "  0%|                                     | 127/48187 [00:25<2:43:20,  4.90it/s]\u001b[A\n",
      "  0%|                                     | 128/48187 [00:25<2:43:27,  4.90it/s]\u001b[A\n",
      "  0%|                                     | 129/48187 [00:26<2:43:06,  4.91it/s]\u001b[A\n",
      "  0%|                                     | 130/48187 [00:26<2:43:20,  4.90it/s]\u001b[A\n",
      "  0%|                                     | 131/48187 [00:26<2:42:27,  4.93it/s]\u001b[A\n",
      "  0%|                                     | 132/48187 [00:26<2:42:53,  4.92it/s]\u001b[A\n",
      "  0%|                                     | 133/48187 [00:26<2:42:41,  4.92it/s]\u001b[A\n",
      "  0%|                                     | 134/48187 [00:27<2:43:15,  4.91it/s]\u001b[A\n",
      "  0%|                                     | 135/48187 [00:27<2:43:40,  4.89it/s]\u001b[A\n",
      "  0%|                                     | 136/48187 [00:27<2:45:50,  4.83it/s]\u001b[A\n",
      "  0%|                                     | 137/48187 [00:27<2:42:53,  4.92it/s]\u001b[A\n",
      "  0%|                                     | 138/48187 [00:27<2:43:47,  4.89it/s]\u001b[A\n",
      "  0%|                                     | 139/48187 [00:28<2:43:38,  4.89it/s]\u001b[A\n",
      "  0%|                                     | 140/48187 [00:28<2:43:06,  4.91it/s]\u001b[A\n",
      "  0%|                                     | 141/48187 [00:28<2:44:42,  4.86it/s]\u001b[A\n",
      "  0%|                                     | 142/48187 [00:28<2:42:33,  4.93it/s]\u001b[A\n",
      "  0%|                                     | 143/48187 [00:28<2:42:28,  4.93it/s]\u001b[A\n",
      "  0%|                                     | 144/48187 [00:29<2:43:19,  4.90it/s]\u001b[A\n",
      "  0%|                                     | 145/48187 [00:29<2:44:03,  4.88it/s]\u001b[A\n",
      "  0%|                                     | 146/48187 [00:29<2:46:22,  4.81it/s]\u001b[A\n",
      "  0%|                                     | 147/48187 [00:29<2:43:09,  4.91it/s]\u001b[A\n",
      "  0%|                                     | 148/48187 [00:29<2:42:53,  4.92it/s]\u001b[A\n",
      "  0%|                                     | 149/48187 [00:30<2:43:35,  4.89it/s]\u001b[A\n",
      "  0%|                                     | 150/48187 [00:30<2:43:22,  4.90it/s]\u001b[A\n",
      "  0%|                                     | 151/48187 [00:30<2:43:40,  4.89it/s]\u001b[A\n",
      "  0%|                                     | 152/48187 [00:30<2:45:15,  4.84it/s]\u001b[A\n",
      "  0%|                                     | 153/48187 [00:31<2:44:27,  4.87it/s]\u001b[A\n",
      "  0%|                                     | 154/48187 [00:31<2:47:45,  4.77it/s]\u001b[A\n",
      "  0%|                                     | 155/48187 [00:31<2:43:03,  4.91it/s]\u001b[A\n",
      "  0%|                                     | 156/48187 [00:31<2:42:56,  4.91it/s]\u001b[A\n",
      "  0%|                                     | 157/48187 [00:31<2:42:43,  4.92it/s]\u001b[A\n",
      "  0%|                                     | 158/48187 [00:32<2:43:12,  4.90it/s]\u001b[A\n",
      "  0%|                                     | 159/48187 [00:32<2:44:50,  4.86it/s]\u001b[A\n",
      "  0%|                                     | 160/48187 [00:32<2:44:07,  4.88it/s]\u001b[A\n",
      "  0%|                                     | 161/48187 [00:32<2:44:38,  4.86it/s]\u001b[A\n",
      "  0%|                                     | 162/48187 [00:32<2:44:06,  4.88it/s]\u001b[A\n",
      "  0%|▏                                    | 163/48187 [00:33<2:44:40,  4.86it/s]\u001b[A\n",
      "  0%|▏                                    | 164/48187 [00:33<2:46:30,  4.81it/s]\u001b[A\n",
      "  0%|▏                                    | 165/48187 [00:33<2:43:52,  4.88it/s]\u001b[A\n",
      "  0%|▏                                    | 166/48187 [00:33<2:43:57,  4.88it/s]\u001b[A\n",
      "  0%|▏                                    | 167/48187 [00:33<2:44:44,  4.86it/s]\u001b[A\n",
      "  0%|▏                                    | 168/48187 [00:34<2:43:54,  4.88it/s]\u001b[A\n",
      "  0%|▏                                    | 169/48187 [00:34<2:43:49,  4.88it/s]\u001b[A\n",
      "  0%|▏                                    | 170/48187 [00:34<2:43:24,  4.90it/s]\u001b[A\n",
      "  0%|▏                                    | 171/48187 [00:34<2:43:27,  4.90it/s]\u001b[A\n",
      "  0%|▏                                    | 172/48187 [00:34<2:43:28,  4.90it/s]\u001b[A\n",
      "  0%|▏                                    | 173/48187 [00:35<2:44:14,  4.87it/s]\u001b[A\n",
      "  0%|▏                                    | 174/48187 [00:35<2:44:01,  4.88it/s]\u001b[A\n",
      "  0%|▏                                    | 175/48187 [00:35<2:44:07,  4.88it/s]\u001b[A\n",
      "  0%|▏                                    | 176/48187 [00:35<2:43:37,  4.89it/s]\u001b[A\n",
      "  0%|▏                                    | 177/48187 [00:35<2:43:41,  4.89it/s]\u001b[A\n",
      "  0%|▏                                    | 178/48187 [00:36<2:44:06,  4.88it/s]\u001b[A\n",
      "  0%|▏                                    | 179/48187 [00:36<2:43:40,  4.89it/s]\u001b[A\n",
      "  0%|▏                                    | 180/48187 [00:36<2:45:30,  4.83it/s]\u001b[A\n",
      "  0%|▏                                    | 181/48187 [00:36<2:45:00,  4.85it/s]\u001b[A\n",
      "  0%|▏                                    | 182/48187 [00:36<2:44:27,  4.86it/s]\u001b[A\n",
      "  0%|▏                                    | 183/48187 [00:37<2:44:24,  4.87it/s]\u001b[A\n",
      "  0%|▏                                    | 184/48187 [00:37<2:46:21,  4.81it/s]\u001b[A\n",
      "  0%|▏                                    | 185/48187 [00:37<2:43:37,  4.89it/s]\u001b[A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|▏                                    | 186/48187 [00:37<2:43:45,  4.89it/s]\u001b[A\n",
      "  0%|▏                                    | 187/48187 [00:38<2:44:38,  4.86it/s]\u001b[A\n",
      "  0%|▏                                    | 188/48187 [00:38<2:43:53,  4.88it/s]\u001b[A\n",
      "  0%|▏                                    | 189/48187 [00:38<2:43:36,  4.89it/s]\u001b[A\n",
      "  0%|▏                                    | 190/48187 [00:38<2:43:38,  4.89it/s]\u001b[A\n",
      "  0%|▏                                    | 191/48187 [00:38<2:43:27,  4.89it/s]\u001b[A\n",
      "  0%|▏                                    | 192/48187 [00:39<2:43:33,  4.89it/s]\u001b[A\n",
      "  0%|▏                                    | 193/48187 [00:39<2:45:20,  4.84it/s]\u001b[A\n",
      "  0%|▏                                    | 194/48187 [00:39<2:43:47,  4.88it/s]\u001b[A\n",
      "  0%|▏                                    | 195/48187 [00:39<2:44:14,  4.87it/s]\u001b[A\n",
      "  0%|▏                                    | 196/48187 [00:39<2:44:39,  4.86it/s]\u001b[A\n",
      "  0%|▏                                    | 197/48187 [00:40<2:44:24,  4.86it/s]\u001b[A\n",
      "  0%|▏                                    | 198/48187 [00:40<2:44:46,  4.85it/s]\u001b[A\n",
      "  0%|▏                                    | 199/48187 [00:40<2:46:14,  4.81it/s]\u001b[A\n",
      "  0%|▏                                    | 200/48187 [00:40<2:44:54,  4.85it/s]\u001b[A\n",
      "  0%|▏                                    | 201/48187 [00:40<2:44:48,  4.85it/s]\u001b[A\n",
      "  0%|▏                                    | 202/48187 [00:41<2:45:41,  4.83it/s]\u001b[A\n",
      "  0%|▏                                    | 203/48187 [00:41<2:43:57,  4.88it/s]\u001b[A\n",
      "  0%|▏                                    | 204/48187 [00:41<2:44:54,  4.85it/s]\u001b[A\n",
      "  0%|▏                                    | 205/48187 [00:41<2:44:42,  4.86it/s]\u001b[A\n",
      "  0%|▏                                    | 206/48187 [00:41<2:44:23,  4.86it/s]\u001b[A\n",
      "  0%|▏                                    | 207/48187 [00:42<2:44:15,  4.87it/s]\u001b[A\n",
      "  0%|▏                                    | 208/48187 [00:42<2:45:24,  4.83it/s]\u001b[A\n",
      "  0%|▏                                    | 209/48187 [00:42<2:44:52,  4.85it/s]\u001b[A\n",
      "  0%|▏                                    | 210/48187 [00:42<2:44:56,  4.85it/s]\u001b[A\n",
      "  0%|▏                                    | 211/48187 [00:42<2:45:04,  4.84it/s]\u001b[A\n",
      "  0%|▏                                    | 212/48187 [00:43<2:45:35,  4.83it/s]\u001b[A\n",
      "  0%|▏                                    | 213/48187 [00:43<2:43:23,  4.89it/s]\u001b[A\n",
      "  0%|▏                                    | 214/48187 [00:43<2:44:35,  4.86it/s]\u001b[A\n",
      "  0%|▏                                    | 215/48187 [00:43<2:44:30,  4.86it/s]\u001b[A\n",
      "  0%|▏                                    | 216/48187 [00:43<2:44:29,  4.86it/s]\u001b[A\n",
      "  0%|▏                                    | 217/48187 [00:44<2:45:47,  4.82it/s]\u001b[A\n",
      "  0%|▏                                    | 218/48187 [00:44<2:43:55,  4.88it/s]\u001b[A\n",
      "  0%|▏                                    | 219/48187 [00:44<2:44:54,  4.85it/s]\u001b[A\n",
      "  0%|▏                                    | 220/48187 [00:44<2:44:54,  4.85it/s]\u001b[A\n",
      "  0%|▏                                    | 221/48187 [00:45<2:44:46,  4.85it/s]\u001b[A\n",
      "  0%|▏                                    | 222/48187 [00:45<2:44:30,  4.86it/s]\u001b[A\n",
      "  0%|▏                                    | 223/48187 [00:45<2:45:43,  4.82it/s]\u001b[A\n",
      "  0%|▏                                    | 224/48187 [00:45<2:44:45,  4.85it/s]\u001b[A\n",
      "  0%|▏                                    | 225/48187 [00:45<2:45:29,  4.83it/s]\u001b[A\n",
      "  0%|▏                                    | 226/48187 [00:46<2:44:47,  4.85it/s]\u001b[A\n",
      "  0%|▏                                    | 227/48187 [00:46<2:44:07,  4.87it/s]\u001b[A\n",
      "  0%|▏                                    | 228/48187 [00:46<2:44:51,  4.85it/s]\u001b[A\n",
      "  0%|▏                                    | 229/48187 [00:46<2:44:50,  4.85it/s]\u001b[A\n",
      "  0%|▏                                    | 230/48187 [00:46<2:44:42,  4.85it/s]\u001b[A\n",
      "  0%|▏                                    | 231/48187 [00:47<2:44:43,  4.85it/s]\u001b[A\n",
      "  0%|▏                                    | 232/48187 [00:47<2:44:40,  4.85it/s]\u001b[A\n",
      "  0%|▏                                    | 233/48187 [00:47<2:47:14,  4.78it/s]\u001b[A\n",
      "  0%|▏                                    | 234/48187 [00:47<2:46:07,  4.81it/s]\u001b[A\n",
      "  0%|▏                                    | 235/48187 [00:47<2:45:08,  4.84it/s]\u001b[A\n",
      "  0%|▏                                    | 236/48187 [00:48<2:45:01,  4.84it/s]\u001b[A\n",
      "  0%|▏                                    | 237/48187 [00:48<2:45:14,  4.84it/s]\u001b[A\n",
      "  0%|▏                                    | 238/48187 [00:48<2:49:08,  4.72it/s]\u001b[A\n",
      "  0%|▏                                    | 239/48187 [00:48<2:44:38,  4.85it/s]\u001b[A\n",
      "  0%|▏                                    | 240/48187 [00:48<2:44:48,  4.85it/s]\u001b[A\n",
      "  1%|▏                                    | 241/48187 [00:49<2:45:17,  4.83it/s]\u001b[A\n",
      "  1%|▏                                    | 242/48187 [00:49<2:45:00,  4.84it/s]\u001b[A\n",
      "  1%|▏                                    | 243/48187 [00:49<2:44:54,  4.85it/s]\u001b[A\n",
      "  1%|▏                                    | 244/48187 [00:49<2:45:34,  4.83it/s]\u001b[A\n",
      "  1%|▏                                    | 245/48187 [00:49<2:45:36,  4.82it/s]\u001b[A\n",
      "  1%|▏                                    | 246/48187 [00:50<2:45:33,  4.83it/s]\u001b[A\n",
      "  1%|▏                                    | 247/48187 [00:50<2:47:01,  4.78it/s]\u001b[A\n",
      "  1%|▏                                    | 248/48187 [00:50<2:44:27,  4.86it/s]\u001b[A\n",
      "  1%|▏                                    | 249/48187 [00:50<2:45:08,  4.84it/s]\u001b[A\n",
      "  1%|▏                                    | 250/48187 [00:51<2:45:07,  4.84it/s]\u001b[A\n",
      "  1%|▏                                    | 251/48187 [00:51<2:45:13,  4.84it/s]\u001b[A\n",
      "  1%|▏                                    | 252/48187 [00:51<2:48:11,  4.75it/s]\u001b[A\n",
      "  1%|▏                                    | 253/48187 [00:51<2:44:30,  4.86it/s]\u001b[A\n",
      "  1%|▏                                    | 254/48187 [00:51<2:45:12,  4.84it/s]\u001b[A\n",
      "  1%|▏                                    | 255/48187 [00:52<2:45:17,  4.83it/s]\u001b[A\n",
      "  1%|▏                                    | 256/48187 [00:52<2:45:00,  4.84it/s]\u001b[A\n",
      "  1%|▏                                    | 257/48187 [00:52<2:47:42,  4.76it/s]\u001b[A\n",
      "  1%|▏                                    | 258/48187 [00:52<2:45:02,  4.84it/s]\u001b[A\n",
      "  1%|▏                                    | 259/48187 [00:52<2:45:10,  4.84it/s]\u001b[A\n",
      "  1%|▏                                    | 260/48187 [00:53<2:46:45,  4.79it/s]\u001b[A\n",
      "  1%|▏                                    | 261/48187 [00:53<2:45:12,  4.83it/s]\u001b[A\n",
      "  1%|▏                                    | 262/48187 [00:53<2:44:52,  4.84it/s]\u001b[A\n",
      "  1%|▏                                    | 263/48187 [00:53<2:45:40,  4.82it/s]\u001b[A\n",
      "  1%|▏                                    | 264/48187 [00:53<2:45:34,  4.82it/s]\u001b[A\n",
      "  1%|▏                                    | 265/48187 [00:54<2:45:17,  4.83it/s]\u001b[A\n",
      "  1%|▏                                    | 266/48187 [00:54<2:45:40,  4.82it/s]\u001b[A\n",
      "  1%|▏                                    | 267/48187 [00:54<2:46:33,  4.80it/s]\u001b[A\n",
      "  1%|▏                                    | 268/48187 [00:54<2:45:28,  4.83it/s]\u001b[A\n",
      "  1%|▏                                    | 269/48187 [00:54<2:45:24,  4.83it/s]\u001b[A\n",
      "  1%|▏                                    | 270/48187 [00:55<2:44:55,  4.84it/s]\u001b[A\n",
      "  1%|▏                                    | 271/48187 [00:55<2:45:43,  4.82it/s]\u001b[A\n",
      "  1%|▏                                    | 272/48187 [00:55<2:45:38,  4.82it/s]\u001b[A\n",
      "  1%|▏                                    | 273/48187 [00:55<2:46:08,  4.81it/s]\u001b[A\n",
      "  1%|▏                                    | 274/48187 [00:55<2:45:33,  4.82it/s]\u001b[A\n",
      "  1%|▏                                    | 275/48187 [00:56<2:45:06,  4.84it/s]\u001b[A\n",
      "  1%|▏                                    | 276/48187 [00:56<2:45:42,  4.82it/s]\u001b[A\n",
      "  1%|▏                                    | 277/48187 [00:56<2:46:05,  4.81it/s]\u001b[A\n",
      "  1%|▏                                    | 278/48187 [00:56<2:48:12,  4.75it/s]\u001b[A\n",
      "  1%|▏                                    | 279/48187 [00:57<2:45:56,  4.81it/s]\u001b[A\n",
      "  1%|▏                                    | 280/48187 [00:57<2:46:39,  4.79it/s]\u001b[A\n",
      "  1%|▏                                    | 281/48187 [00:57<2:46:22,  4.80it/s]\u001b[A\n",
      "  1%|▏                                    | 282/48187 [00:57<2:45:56,  4.81it/s]\u001b[A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  1%|▏                                    | 283/48187 [00:57<2:46:58,  4.78it/s]\u001b[A\n",
      "  1%|▏                                    | 284/48187 [00:58<2:46:29,  4.80it/s]\u001b[A\n",
      "  1%|▏                                    | 285/48187 [00:58<2:46:01,  4.81it/s]\u001b[A\n",
      "  1%|▏                                    | 286/48187 [00:58<2:46:16,  4.80it/s]\u001b[A\n",
      "  1%|▏                                    | 287/48187 [00:58<2:45:32,  4.82it/s]\u001b[A\n",
      "  1%|▏                                    | 288/48187 [00:58<2:45:26,  4.83it/s]\u001b[A\n",
      "  1%|▏                                    | 289/48187 [00:59<2:47:42,  4.76it/s]\u001b[A\n",
      "  1%|▏                                    | 290/48187 [00:59<2:45:56,  4.81it/s]\u001b[A\n",
      "  1%|▏                                    | 291/48187 [00:59<2:45:15,  4.83it/s]\u001b[A\n",
      "  1%|▏                                    | 292/48187 [00:59<2:45:03,  4.84it/s]\u001b[A\n",
      "  1%|▏                                    | 293/48187 [00:59<2:45:24,  4.83it/s]\u001b[A\n",
      "  1%|▏                                    | 294/48187 [01:00<2:45:42,  4.82it/s]\u001b[A\n",
      "  1%|▏                                    | 295/48187 [01:00<2:45:34,  4.82it/s]\u001b[A\n",
      "  1%|▏                                    | 296/48187 [01:00<2:45:27,  4.82it/s]\u001b[A\n",
      "  1%|▏                                    | 297/48187 [01:00<2:45:08,  4.83it/s]\u001b[A\n",
      "  1%|▏                                    | 298/48187 [01:00<2:47:19,  4.77it/s]\u001b[A\n",
      "  1%|▏                                    | 299/48187 [01:01<2:45:25,  4.82it/s]\u001b[A\n",
      "  1%|▏                                    | 300/48187 [01:01<2:45:35,  4.82it/s]\u001b[A\n",
      "  1%|▏                                    | 301/48187 [01:01<2:45:15,  4.83it/s]\u001b[A\n",
      "  1%|▏                                    | 302/48187 [01:01<2:45:52,  4.81it/s]\u001b[A\n",
      "  1%|▏                                    | 303/48187 [01:02<2:45:50,  4.81it/s]\u001b[A\n",
      "  1%|▏                                    | 304/48187 [01:02<2:45:40,  4.82it/s]\u001b[A\n",
      "  1%|▏                                    | 305/48187 [01:02<2:45:53,  4.81it/s]\u001b[A\n",
      "  1%|▏                                    | 306/48187 [01:02<2:47:12,  4.77it/s]\u001b[A\n",
      "  1%|▏                                    | 307/48187 [01:02<2:46:03,  4.81it/s]\u001b[A\n",
      "  1%|▏                                    | 308/48187 [01:03<2:46:02,  4.81it/s]\u001b[A\n",
      "  1%|▏                                    | 309/48187 [01:03<2:46:41,  4.79it/s]\u001b[A\n",
      "  1%|▏                                    | 310/48187 [01:03<2:46:28,  4.79it/s]\u001b[A\n",
      "  1%|▏                                    | 311/48187 [01:03<2:48:36,  4.73it/s]\u001b[A\n",
      "  1%|▏                                    | 312/48187 [01:03<2:45:46,  4.81it/s]\u001b[A\n",
      "  1%|▏                                    | 313/48187 [01:04<2:46:10,  4.80it/s]\u001b[A\n",
      "  1%|▏                                    | 314/48187 [01:04<2:46:37,  4.79it/s]\u001b[A\n",
      "  1%|▏                                    | 315/48187 [01:04<2:46:05,  4.80it/s]\u001b[A\n",
      "  1%|▏                                    | 316/48187 [01:04<2:46:18,  4.80it/s]\u001b[A\n",
      "  1%|▏                                    | 317/48187 [01:04<2:45:45,  4.81it/s]\u001b[A\n",
      "  1%|▏                                    | 318/48187 [01:05<2:46:39,  4.79it/s]\u001b[A\n",
      "  1%|▏                                    | 319/48187 [01:05<2:47:14,  4.77it/s]\u001b[A\n",
      "  1%|▏                                    | 320/48187 [01:05<2:46:24,  4.79it/s]\u001b[A\n",
      "  1%|▏                                    | 321/48187 [01:05<2:45:37,  4.82it/s]\u001b[A\n",
      "  1%|▏                                    | 322/48187 [01:05<2:46:17,  4.80it/s]\u001b[A\n",
      "  1%|▏                                    | 323/48187 [01:06<2:46:45,  4.78it/s]\u001b[A\n",
      "  1%|▏                                    | 324/48187 [01:06<2:46:07,  4.80it/s]\u001b[A\n",
      "  1%|▏                                    | 325/48187 [01:06<2:48:19,  4.74it/s]\u001b[A\n",
      "  1%|▎                                    | 326/48187 [01:06<2:45:54,  4.81it/s]\u001b[A\n",
      "  1%|▎                                    | 327/48187 [01:07<2:46:38,  4.79it/s]\u001b[A\n",
      "  1%|▎                                    | 328/48187 [01:07<2:46:29,  4.79it/s]\u001b[A\n",
      "  1%|▎                                    | 329/48187 [01:07<2:46:06,  4.80it/s]\u001b[A\n",
      "  1%|▎                                    | 330/48187 [01:07<2:49:02,  4.72it/s]\u001b[A\n",
      "  1%|▎                                    | 331/48187 [01:07<2:46:04,  4.80it/s]\u001b[A\n",
      "  1%|▎                                    | 332/48187 [01:08<2:46:25,  4.79it/s]\u001b[A\n",
      "  1%|▎                                    | 333/48187 [01:08<2:46:27,  4.79it/s]\u001b[A\n",
      "  1%|▎                                    | 334/48187 [01:08<2:46:11,  4.80it/s]\u001b[A\n",
      "  1%|▎                                    | 335/48187 [01:08<2:47:01,  4.77it/s]\u001b[A\n",
      "  1%|▎                                    | 336/48187 [01:08<2:46:44,  4.78it/s]\u001b[A\n",
      "  1%|▎                                    | 337/48187 [01:09<2:46:05,  4.80it/s]\u001b[A\n",
      "  1%|▎                                    | 338/48187 [01:09<2:46:01,  4.80it/s]\u001b[A\n",
      "  1%|▎                                    | 339/48187 [01:09<2:47:08,  4.77it/s]\u001b[A\n",
      "  1%|▎                                    | 340/48187 [01:09<2:46:44,  4.78it/s]\u001b[A\n",
      "  1%|▎                                    | 341/48187 [01:09<2:46:45,  4.78it/s]\u001b[A\n",
      "  1%|▎                                    | 342/48187 [01:10<2:46:05,  4.80it/s]\u001b[A\n",
      "  1%|▎                                    | 343/48187 [01:10<2:46:19,  4.79it/s]\u001b[A\n",
      "  1%|▎                                    | 344/48187 [01:10<2:46:07,  4.80it/s]\u001b[A\n",
      "  1%|▎                                    | 345/48187 [01:10<2:46:01,  4.80it/s]\u001b[A\n",
      "  1%|▎                                    | 346/48187 [01:10<2:45:48,  4.81it/s]\u001b[A\n",
      "  1%|▎                                    | 347/48187 [01:11<2:46:11,  4.80it/s]\u001b[A\n",
      "  1%|▎                                    | 348/48187 [01:11<2:46:37,  4.79it/s]\u001b[A\n",
      "  1%|▎                                    | 349/48187 [01:11<2:46:53,  4.78it/s]\u001b[A\n",
      "  1%|▎                                    | 350/48187 [01:11<2:46:30,  4.79it/s]\u001b[A\n",
      "  1%|▎                                    | 351/48187 [01:12<2:46:53,  4.78it/s]\u001b[A\n",
      "  1%|▎                                    | 352/48187 [01:12<2:47:11,  4.77it/s]\u001b[A\n",
      "  1%|▎                                    | 353/48187 [01:12<2:47:12,  4.77it/s]\u001b[A\n",
      "  1%|▎                                    | 354/48187 [01:12<2:46:29,  4.79it/s]\u001b[A\n",
      "  1%|▎                                    | 355/48187 [01:12<2:48:19,  4.74it/s]\u001b[A\n",
      "  1%|▎                                    | 356/48187 [01:13<2:44:04,  4.86it/s]\u001b[A\n",
      "  0%|                                               | 0/2567 [01:13<?, ?batch/s]\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn [10], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mevaluator\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43meval\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/data/home/admin/t5-experiments-from-yura/venv/lib/python3.8/site-packages/torch/autograd/grad_mode.py:27\u001b[0m, in \u001b[0;36m_DecoratorContextManager.__call__.<locals>.decorate_context\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     24\u001b[0m \u001b[38;5;129m@functools\u001b[39m\u001b[38;5;241m.\u001b[39mwraps(func)\n\u001b[1;32m     25\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdecorate_context\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m     26\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mclone():\n\u001b[0;32m---> 27\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "Cell \u001b[0;32mIn [8], line 71\u001b[0m, in \u001b[0;36mEvaluator.eval\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     66\u001b[0m chunk_end \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mmin\u001b[39m(\n\u001b[1;32m     67\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mchunk_size \u001b[38;5;241m*\u001b[39m (chunk_number \u001b[38;5;241m+\u001b[39m \u001b[38;5;241m1\u001b[39m), \u001b[38;5;28mlen\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mentity_strings)\n\u001b[1;32m     68\u001b[0m )\n\u001b[1;32m     69\u001b[0m current_chunk_size \u001b[38;5;241m=\u001b[39m chunk_end \u001b[38;5;241m-\u001b[39m chunk_start\n\u001b[0;32m---> 71\u001b[0m input_ids_repeated \u001b[38;5;241m=\u001b[39m \u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrepeat_interleave\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m     72\u001b[0m \u001b[43m    \u001b[49m\u001b[43minput_ids\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcurrent_chunk_size\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdim\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m0\u001b[39;49m\n\u001b[1;32m     73\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     75\u001b[0m attention_mask_repeated \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mrepeat_interleave(\n\u001b[1;32m     76\u001b[0m     attention_mask, current_chunk_size, dim\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m0\u001b[39m\n\u001b[1;32m     77\u001b[0m )\n\u001b[1;32m     79\u001b[0m entities_repeated_chunk \u001b[38;5;241m=\u001b[39m \\\n\u001b[1;32m     80\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdataset\u001b[38;5;241m.\u001b[39mtokenizer\u001b[38;5;241m.\u001b[39mbatch_encode_plus(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mentity_strings[chunk_start:chunk_end], padding\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mmax_length\u001b[39m\u001b[38;5;124m'\u001b[39m, \n\u001b[1;32m     81\u001b[0m     truncation\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, max_length\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m32\u001b[39m, return_tensors\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpt\u001b[39m\u001b[38;5;124m\"\u001b[39m)\u001b[38;5;241m.\u001b[39minput_ids\u001b[38;5;241m.\u001b[39mto(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdevice)\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "evaluator.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4de7f509",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "venv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}