notebooks/Simple QA Models/Relation Classifier RNN Model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Relation Classifier RNN Model\n",
    "\n",
    "Our goal here is to implement a relation model. Our goal is to do basic relation classification.\n",
    "\n",
    "First things first, set up the initial configuration."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.insert(0, '../../')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f5cf282a2e7049a3986a8743299c36fe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm_notebook\n",
    "from functools import partial\n",
    "tqdm_notebook = partial(tqdm_notebook, leave=False)\n",
    "tqdm_notebook().pandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Experiment Folder: ../../experiments/relation_classifier.02_02_13:31:11\n",
      "[2018-02-02 13:31:11,040][MainProcess][__main__][INFO] Device: 0\n",
      "[2018-02-02 13:31:11,041][MainProcess][lib.utils][INFO] Seed: 1212212\n"
     ]
    }
   ],
   "source": [
    "import logging\n",
    "\n",
    "from lib.utils import config_logging\n",
    "from lib.utils import new_experiment_folder\n",
    "from lib.utils import seed\n",
    "from lib.utils import save_standard_streams\n",
    "from lib.utils import device_default\n",
    "\n",
    "experiment_folder = new_experiment_folder(label='relation_classifier', parent_directory='../../experiments/')\n",
    "print('Experiment Folder: %s' % experiment_folder)\n",
    "save_standard_streams(experiment_folder) # Copy Stdout and Stderr to experiments folder\n",
    "\n",
    "config_logging()\n",
    "logger = logging.getLogger(__name__)\n",
    "\n",
    "device = device_default()\n",
    "logger.info('Device: %d', device)\n",
    "is_cuda = device >= 0\n",
    "cuda = lambda t: t.cuda(device=device) if is_cuda else t\n",
    "\n",
    "random_seed = 1212212\n",
    "seed(random_seed, is_cuda=is_cuda)\n",
    "\n",
    "# Async minibatch allocation for speed\n",
    "# Reference: http://timdettmers.com/2015/03/09/deep-learning-hardware-guide/\n",
    "cuda_async = lambda t: t.cuda(device=device, async=True) if is_cuda else t"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset\n",
    "\n",
    "Load our dataset. Log a couple rows."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "from tqdm import tqdm_notebook\n",
    "from torchnlp.datasets import Dataset\n",
    "\n",
    "def simple_qa_predicate(directory='../../data/relation_ranking/',\n",
    "               train=False,\n",
    "               dev=False,\n",
    "               test=False,\n",
    "               train_filename='train.txt',\n",
    "               dev_filename='dev.txt',\n",
    "               test_filename=''):\n",
    "    \"\"\"\n",
    "    Example line example: \n",
    "        film/film/country\tfilm/film/country film/film/genre film/film/language\twhat country is <e> from ?\n",
    "    Vocab example: \n",
    "        /film/film/genre\n",
    "    \n",
    "    Sample Data:\n",
    "        Question: 'which genre of album is #head_entity# ?'\n",
    "        True Relation: '/music/album/genre'\n",
    "        False Relation Pool: ['/music/album/release_type', '/music/album/genre', '/music/album/artist']\n",
    "    \"\"\"\n",
    "    ret = []\n",
    "    datasets = [(train, train_filename), (dev, dev_filename), (test, test_filename)]\n",
    "    for is_requested, filename in datasets:\n",
    "        if not is_requested:\n",
    "            continue\n",
    "            \n",
    "        file_path = os.path.join(directory, filename)\n",
    "        data = pd.read_table(file_path, header=None, names=['True Relation', 'Relation Pool', 'Question', 'Entity'])\n",
    "        rows = []\n",
    "        for i, row in tqdm_notebook(data.iterrows(), total=data.shape[0]):\n",
    "            relation_pool = list(set(row['Relation Pool'].split()))\n",
    "            rows.append({\n",
    "                'text': row['Question'].strip(),\n",
    "                'relation': row['True Relation'],\n",
    "                'pool': relation_pool,\n",
    "                'entity': row['Entity'].strip()})\n",
    "            \n",
    "        ret.append(Dataset(rows))\n",
    "\n",
    "    if len(ret) == 1:\n",
    "        return ret[0]\n",
    "    else:\n",
    "        return tuple(ret)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b92774bea27b46589da91a09223b1d52",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=75729), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d1937535e3714fe6b796e06aabe111b8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=10816), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Num Training Data: 75729\n",
      "Train Sample:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>entity</th>\n",
       "      <th>pool</th>\n",
       "      <th>relation</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>e</td>\n",
       "      <td>[music/album/genre, language/human_language/la...</td>\n",
       "      <td>book/written_work/subjects</td>\n",
       "      <td>what is the book &lt;e&gt; about</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>cardiac arrest</td>\n",
       "      <td>[music/artist/track, music/album/genre, music/...</td>\n",
       "      <td>music/release_track/release</td>\n",
       "      <td>to what release does the release track &lt;e&gt; com...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>the debt</td>\n",
       "      <td>[film/film/production_companies, music/album/a...</td>\n",
       "      <td>film/film/country</td>\n",
       "      <td>what country was the film &lt;e&gt; from</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>nobuo uematsu</td>\n",
       "      <td>[music/artist/track, organization/organization...</td>\n",
       "      <td>music/producer/tracks_produced</td>\n",
       "      <td>what songs have &lt;e&gt; produced ?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>eve - olution</td>\n",
       "      <td>[music/album/genre, music/release/region, musi...</td>\n",
       "      <td>music/release/producers</td>\n",
       "      <td>who produced &lt;e&gt; ?</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           entity                                               pool  \\\n",
       "0               e  [music/album/genre, language/human_language/la...   \n",
       "1  cardiac arrest  [music/artist/track, music/album/genre, music/...   \n",
       "2        the debt  [film/film/production_companies, music/album/a...   \n",
       "3   nobuo uematsu  [music/artist/track, organization/organization...   \n",
       "4   eve - olution  [music/album/genre, music/release/region, musi...   \n",
       "\n",
       "                         relation  \\\n",
       "0      book/written_work/subjects   \n",
       "1     music/release_track/release   \n",
       "2               film/film/country   \n",
       "3  music/producer/tracks_produced   \n",
       "4         music/release/producers   \n",
       "\n",
       "                                                text  \n",
       "0                         what is the book <e> about  \n",
       "1  to what release does the release track <e> com...  \n",
       "2                 what country was the film <e> from  \n",
       "3                     what songs have <e> produced ?  \n",
       "4                                 who produced <e> ?  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num Development Data: 10816\n",
      "Development Sample:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>entity</th>\n",
       "      <th>pool</th>\n",
       "      <th>relation</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>american</td>\n",
       "      <td>[people/ethnicity/geographic_distribution, peo...</td>\n",
       "      <td>biology/organism_classification/organisms_of_t...</td>\n",
       "      <td>name an &lt;e&gt; thoroughbread racehorse</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>vision racing driving simulator</td>\n",
       "      <td>[cvg/computer_videogame/cvg_genre, cvg/compute...</td>\n",
       "      <td>cvg/computer_videogame/cvg_genre</td>\n",
       "      <td>what kind of game is &lt;e&gt; ?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>romance film</td>\n",
       "      <td>[base/skosbase/vocabulary_equivalent_topic/bro...</td>\n",
       "      <td>tv/tv_genre/programs</td>\n",
       "      <td>what tv program is &lt;e&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>polaski</td>\n",
       "      <td>[location/location/containedby]</td>\n",
       "      <td>location/location/containedby</td>\n",
       "      <td>what state is &lt;e&gt; located in</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>fern emmett</td>\n",
       "      <td>[people/person/gender, people/person/nationali...</td>\n",
       "      <td>people/deceased_person/cause_of_death</td>\n",
       "      <td>what disease claimed the life of &lt;e&gt;</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            entity  \\\n",
       "0                         american   \n",
       "1  vision racing driving simulator   \n",
       "2                     romance film   \n",
       "3                          polaski   \n",
       "4                      fern emmett   \n",
       "\n",
       "                                                pool  \\\n",
       "0  [people/ethnicity/geographic_distribution, peo...   \n",
       "1  [cvg/computer_videogame/cvg_genre, cvg/compute...   \n",
       "2  [base/skosbase/vocabulary_equivalent_topic/bro...   \n",
       "3                    [location/location/containedby]   \n",
       "4  [people/person/gender, people/person/nationali...   \n",
       "\n",
       "                                            relation  \\\n",
       "0  biology/organism_classification/organisms_of_t...   \n",
       "1                   cvg/computer_videogame/cvg_genre   \n",
       "2                               tv/tv_genre/programs   \n",
       "3                      location/location/containedby   \n",
       "4              people/deceased_person/cause_of_death   \n",
       "\n",
       "                                   text  \n",
       "0   name an <e> thoroughbread racehorse  \n",
       "1            what kind of game is <e> ?  \n",
       "2                what tv program is <e>  \n",
       "3          what state is <e> located in  \n",
       "4  what disease claimed the life of <e>  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_dataset, dev_dataset = simple_qa_predicate(train=True, dev=True, test=False)\n",
    "\n",
    "print('Num Training Data: %d' % len(train_dataset))\n",
    "print('Train Sample:')\n",
    "display(pd.DataFrame(train_dataset[:5]))\n",
    "print('Num Development Data: %d' % len(dev_dataset))\n",
    "print('Development Sample:')\n",
    "display(pd.DataFrame(dev_dataset[:5]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset Iterators\n",
    "\n",
    "Load our dataset. Log a couple rows."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch.autograd import Variable\n",
    "from torchnlp.utils import pad_batch\n",
    "\n",
    "# Defines how to combine a batch of rows into a tensor\n",
    "def collate_fn(batch, train=True):\n",
    "    \"\"\" list of tensors to a batch variable \"\"\"\n",
    "    # PyTorch RNN requires sorting decreasing size\n",
    "    batch = sorted(batch, key=lambda row: len(row['text']), reverse=True)\n",
    "    input_batch, _ = pad_batch([row['text'] for row in batch])\n",
    "    relations = [row['relation'] for row in batch]\n",
    "    mask = Variable(torch.stack([row['mask'] for row in batch]), volatile=not train)\n",
    "\n",
    "    # PyTorch RNN requires batches to be transposed for speed and integration with CUDA\n",
    "    to_variable = (\n",
    "        lambda b: Variable(torch.stack(b).t_().squeeze(0).contiguous(), volatile=not train))\n",
    "\n",
    "    return (to_variable(input_batch), to_variable(relations), mask)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from functools import partial\n",
    "from torchnlp.samplers import NoisySortedBatchSampler\n",
    "from torchnlp.samplers import SortedSampler\n",
    "\n",
    "\n",
    "sort_key = lambda r: r['text'].size()[0]\n",
    "\n",
    "def get_iterator(dataset, batch_size, train=False):\n",
    "    # Use bucket sampling to group similar sized text but with noise + random\n",
    "    batch_sampler = NoisySortedBatchSampler(dataset, batch_size, sort_key, sort_key_noise=0.5)\n",
    "    return DataLoader(\n",
    "        dataset,\n",
    "        batch_sampler=batch_sampler,\n",
    "        collate_fn=partial(collate_fn, train=train),\n",
    "        pin_memory=is_cuda,\n",
    "        num_workers=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Encode Text\n",
    "\n",
    "Here we encode our data into a numerical format. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Text encoder vocab size: 6693\n",
      "Relation encoder vocab size: 1702\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b615aa331f954766abc82245938eccb4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=75729), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c0d09f978e294bc9b2c1545ea985f979",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=10816), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from torchnlp.text_encoders import IdentityEncoder\n",
    "from torchnlp.text_encoders import WhitespaceEncoder\n",
    "\n",
    "# NOTE: Given the development vocab might be cheating if we use any statistics from that...\n",
    "text_data = [t.lower() for t in train_dataset['text'] + dev_dataset['text']]\n",
    "text_encoder = WhitespaceEncoder(text_data, append_eos=False)\n",
    "print('Text encoder vocab size: %d' % text_encoder.vocab_size)\n",
    "relation_encoder = IdentityEncoder(train_dataset['relation'] + dev_dataset['relation'])\n",
    "print('Relation encoder vocab size: %d' % relation_encoder.vocab_size)\n",
    "\n",
    "for dataset in [train_dataset, dev_dataset]:\n",
    "    for row in tqdm_notebook(dataset):\n",
    "        row['text'] = text_encoder.encode(row['text'])\n",
    "        row['relation'] = relation_encoder.encode(row['relation'])\n",
    "        row['pool'] = [relation_encoder.encode(r)[0] for r in row['pool']]\n",
    "        pool = set(row['pool'])\n",
    "        row['mask'] = torch.FloatTensor([1 if i in pool else 0 for i in range(relation_encoder.vocab_size)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model\n",
    "\n",
    "Instantiate the model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2018-02-02 13:31:34,462][MainProcess][lib.pretrained_embeddings][INFO] Loading vectors from ./../../.pretrained_embeddings_cache/wiki.en.vec.pt\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from torchnlp.word_to_vector import FastText\n",
    "\n",
    "# Load embeddings\n",
    "unk_init = lambda t: torch.FloatTensor(t).uniform_(-0.1, 0.1)\n",
    "vocab = set(text_encoder.vocab)\n",
    "pretrained_embedding = FastText(language='en',\n",
    "                                cache='./../../.pretrained_embeddings_cache',\n",
    "                                is_include=lambda w: w in vocab)\n",
    "embedding_weights = torch.Tensor(text_encoder.vocab_size, pretrained_embedding.dim)\n",
    "for i, token in enumerate(text_encoder.vocab):\n",
    "    embedding_weights[i] = pretrained_embedding[token]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from lib.nn import SeqToLabel\n",
    "\n",
    "def make_model():\n",
    "    # `embedding_size=300` is required by FastText\n",
    "    model = SeqToLabel(text_encoder.vocab_size, relation_encoder.vocab_size, embedding_size=300)\n",
    "    for param in model.parameters():\n",
    "        param.data.uniform_(-0.1, 0.1)\n",
    "\n",
    "    model.encoder.embedding.weight.data.copy_(embedding_weights)\n",
    "\n",
    "    cuda(model)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate\n",
    "\n",
    "Instantiate a function that evaluates a model. Our evaluation is the same as \"Step 3 - Predict Relation and Finish\" to ensure consistency."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm_notebook\n",
    "from torch.autograd import Variable\n",
    "import random\n",
    "import math\n",
    "from lib.utils import batch\n",
    "from torchnlp.utils import pad_batch\n",
    "\n",
    "# NOTE: Relation Ranking dataset has a pool of possible relations similar to `Step 3 - Predict Relation and Finish`\n",
    "\n",
    "def get_softmax_relation_score(model, question_batch, mask_batch, relation_pool_batch):\n",
    "    question_batch, _ = pad_batch(question_batch)\n",
    "    question_batch = cuda(Variable(torch.stack(question_batch).t_().squeeze(0).contiguous(), volatile=True))\n",
    "    mask_batch = cuda(torch.stack(mask_batch))\n",
    "    output = model(question_batch, Variable(mask_batch, volatile=True)).exp_().data\n",
    "    \n",
    "    batch_size = output.size()[0]\n",
    "    ret = []\n",
    "    for i in range(batch_size):\n",
    "        ret.append([output[i][j] for j in relation_pool_batch[i]])\n",
    "    return ret\n",
    "\n",
    "def evaluate_pool(dataset, model, batch_size=1):\n",
    "    logger.info('Evaluating...')\n",
    "    \n",
    "    model.train(False)\n",
    "    scores = []\n",
    "    for batched in tqdm_notebook(batch(dataset, batch_size), total=math.ceil(len(dataset) / batch_size)):\n",
    "        question_batch = [row['text'] for row in batched]\n",
    "        mask_batch = [row['mask'] for row in batched]\n",
    "        relation_pool_batch = [row['pool'] for row in batched]\n",
    "        scores.extend(get_softmax_relation_score(model, question_batch, mask_batch, relation_pool_batch))\n",
    "    model.train(True) # No side affects\n",
    "    \n",
    "    correct = 0\n",
    "    for i, row in tqdm_notebook(enumerate(dataset)):\n",
    "        max_score = max(scores[i])\n",
    "        # NOTE: Pool may have multiple <unk> tags\n",
    "        top_relations = set([r for j, r in enumerate(row['pool']) if scores[i][j] == max_score])\n",
    "        predicted_relation = random.choice(list(top_relations))\n",
    "        if predicted_relation == row['relation'][0]:\n",
    "            correct += 1\n",
    "\n",
    "    accuracy = correct / len(dataset)\n",
    "    logger.info('Relation Accuracy (SOTA 88.4%%): %f [%d of %d]', accuracy, correct, len(dataset))\n",
    "    return accuracy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Similar evaluation metric to our training regime. Training the loss should directly affect this metric."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from lib.metrics import print_random_sample\n",
    "from lib.metrics import get_accuracy\n",
    "from torch.nn.modules.loss import NLLLoss\n",
    "\n",
    "def evaluate_softmax(dataset, model, batch_size=1):\n",
    "    # Evaluate\n",
    "    model.train(mode=False)\n",
    "    criterion = cuda(NLLLoss())\n",
    "    texts, relations, outputs = [], [], []\n",
    "    total_loss = 0\n",
    "    dev_iterator = get_iterator(dataset, batch_size)\n",
    "    for text, relation, mask in tqdm_notebook(dev_iterator):\n",
    "        output = model(cuda_async(text), cuda_async(mask))\n",
    "        # Compute metrics\n",
    "        total_loss += criterion(output, cuda_async(relation)).data[0] * relation.size()[0]\n",
    "        # Prevent memory leak by moving output from variable to tensor\n",
    "        texts.extend(text.data.cpu().transpose(0, 1).split(split_size=1, dim=0))\n",
    "        relations.extend(relation.data.cpu().split(split_size=1, dim=0))\n",
    "        outputs.extend(output.data.cpu().split(split_size=1, dim=0))\n",
    "    model.train(True) # No side affects\n",
    "    # Print metrics\n",
    "    # print_random_sample(texts, relations, outputs, text_encoder, relation_encoder, n_samples=5)\n",
    "    logger.info('NLLLoss: %.03f', (total_loss / len(dataset)))\n",
    "    return get_accuracy(relations, outputs, print_=True)[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hyperparameters\n",
    "\n",
    "Set up the hyperparameters for our model. We use `lib.configurable` to setup arguments."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2018-02-02 13:31:38,414][MainProcess][lib.configurable][INFO] Checking configuration...\n",
      "[2018-02-02 13:31:38,415][MainProcess][lib.configurable][WARNING] Path ['lib', 'optim', 'Optimizer', '__init__', 'max_grad_norm'] does not contain @configurable.\n",
      "NOTE: Due to Python remaining the __main__ module, this check can be ignored here.\n",
      "NOTE: _check_configuration can be ignored for external libraries.\n",
      "[2018-02-02 13:31:38,416][MainProcess][lib.configurable][INFO] Configuration checked.\n",
      "[2018-02-02 13:31:38,417][MainProcess][lib.configurable][INFO] Global configuration:\n",
      "[2018-02-02 13:31:38,419][MainProcess][root][INFO] {   'lib': {   'nn': {   'seq_to_label': {   'SeqToLabel': {   '__init__': {   'bidirectional': True,\n",
      "                                                                               'decode_dropout': 0.6,\n",
      "                                                                               'embedding_dropout': 0.3,\n",
      "                                                                               'embedding_size': 300,\n",
      "                                                                               'freeze_embeddings': True,\n",
      "                                                                               'rnn_cell': 'gru',\n",
      "                                                                               'rnn_dropout': 0.8,\n",
      "                                                                               'rnn_layers': 1,\n",
      "                                                                               'rnn_size': 450,\n",
      "                                                                               'rnn_variational_dropout': 0.0}}}},\n",
      "               'optim': {   'Optimizer': {'__init__': {'max_grad_norm': None}},\n",
      "                            'adam': {'Adam': {'__init__': {'amsgrad': True}}}}}}\n"
     ]
    }
   ],
   "source": [
    "from lib.optim import Adam\n",
    "from lib.configurable import configurable\n",
    "from lib.configurable import add_config\n",
    "from lib.configurable import log_config\n",
    "\n",
    "Adam.__init__ = configurable(Adam.__init__)\n",
    "\n",
    "# Hyperparameters achieve 0.881564349112426%\n",
    "# Following the evaluation of 150 random models\n",
    "\n",
    "add_config({\n",
    "    'lib': {\n",
    "        'nn.seq_to_label.SeqToLabel.__init__': {\n",
    "            'bidirectional': True,\n",
    "            'embedding_size': 300,\n",
    "            'rnn_size': 450,\n",
    "            'freeze_embeddings': True,\n",
    "            'rnn_cell': 'gru',\n",
    "            'decode_dropout': 0.6,  # dropout before fully connected layer in RNN\n",
    "            'rnn_layers': 1,\n",
    "            'rnn_variational_dropout': 0.0,\n",
    "            'rnn_dropout': 0.8,\n",
    "            'embedding_dropout': 0.3\n",
    "        },\n",
    "        'optim.Optimizer.__init__.max_grad_norm': None,\n",
    "    },\n",
    "    'lib.optim.adam.Adam.__init__': {\n",
    "        'amsgrad': True\n",
    "    }\n",
    "})\n",
    "\n",
    "log_config()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training Loop\n",
    "\n",
    "Below here, we do a training loop over a number of epochs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from tqdm import tqdm_notebook\n",
    "import random\n",
    "\n",
    "from torch.utils.data import DataLoader\n",
    "from torch.nn.modules.loss import NLLLoss\n",
    "from lib.optim import Adam\n",
    "from functools import partial\n",
    "\n",
    "from lib.checkpoint import Checkpoint\n",
    "from lib.utils import get_total_parameters\n",
    "from lib.utils import resplit_datasets\n",
    "from lib.optimizer import Optimizer\n",
    "\n",
    "# TODO: Try to concat multiple templated questions together and do a multi label classifier.\n",
    "\n",
    "def train(resources=30, checkpoint=None, **kwargs):\n",
    "    \n",
    "    if isinstance(checkpoint, str):\n",
    "        checkpoint = Checkpoint(checkpoint)\n",
    "        model = checkpoint.model\n",
    "        train_batch_size = checkpoint.train_batch_size\n",
    "        optimizer = checkpoint.optimizer\n",
    "        n_bad_epochs = checkpoint.n_bad_epochs\n",
    "        max_score = checkpoint.max_score\n",
    "    else:\n",
    "        model = make_model()\n",
    "        train_batch_size = 32\n",
    "        # NOTE: https://github.com/pytorch/pytorch/issues/679\n",
    "        params = filter(lambda p: p.requires_grad, model.parameters())\n",
    "        optimizer = Optimizer(Adam(params=params))\n",
    "        n_bad_epochs = 0\n",
    "        max_score = 0\n",
    "\n",
    "    # NOTE: Because the training dataset was used to train the subject recongition, its better; therefore,\n",
    "    # we cannot mix them\n",
    "    epochs = max(round(resources), 1)\n",
    "    train_max_batch_size = 1024\n",
    "    patience = 3\n",
    "    criterion = cuda(NLLLoss())\n",
    "    logger.info('Epochs: %d', epochs)\n",
    "    logger.info('Train Dataset Size: %d', len(train_dataset))\n",
    "    logger.info('Dev Dataset Size: %d', len(dev_dataset))\n",
    "    logger.info('Train Batch Size: %d', train_batch_size)\n",
    "    logger.info('Train Max Batch Size: %d', train_max_batch_size)\n",
    "    logger.info('Total Parameters: %d', get_total_parameters(model))\n",
    "    logger.info('Model:\\n%s' % model)\n",
    "\n",
    "    # Train!\n",
    "    for epoch in range(epochs):\n",
    "        logger.info('Epoch: %d', epoch)\n",
    "\n",
    "        # Iterate over the training data\n",
    "        logger.info('Training...')\n",
    "        model.train(mode=True)\n",
    "        train_iterator = get_iterator(train_dataset, train_batch_size, train=True)\n",
    "        for text, relation, mask in tqdm_notebook(train_iterator):\n",
    "            optimizer.zero_grad()\n",
    "            output = model(cuda_async(text), cuda_async(mask))\n",
    "            loss = criterion(output, cuda_async(relation))\n",
    "\n",
    "            # Backward propagation\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "        # Save checkpoint\n",
    "        checkpoint_path = Checkpoint.save(\n",
    "            experiment_folder,\n",
    "            {\n",
    "                'model': model,\n",
    "                'optimizer': optimizer,\n",
    "                'relation_encoder': relation_encoder,\n",
    "                'text_encoder': text_encoder,\n",
    "                'train_batch_size': train_batch_size,\n",
    "                'n_bad_epochs': n_bad_epochs,\n",
    "                'max_score': max_score\n",
    "            },\n",
    "            device=device)\n",
    "\n",
    "        # Evaluate\n",
    "        score = evaluate_softmax(dev_dataset, model, 4096)\n",
    "\n",
    "        # Scheduler for increasing batch_size inspired by this paper:\n",
    "        # https://openreview.net/forum?id=B1Yy1BxCZ\n",
    "        if max_score > score:\n",
    "            n_bad_epochs += 1\n",
    "        else:\n",
    "            n_bad_epochs = 0\n",
    "            max_score = score\n",
    "\n",
    "        if n_bad_epochs >= patience:\n",
    "            train_batch_size = min(train_max_batch_size, train_batch_size * 2)\n",
    "            logger.info('Ran out of patience, increasing train batch size to: %d', train_batch_size)\n",
    "\n",
    "        print('–' * 100)\n",
    "        \n",
    "    return -max_score, checkpoint_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2018-02-02 13:31:38,577][MainProcess][lib.configurable][INFO] seq_to_label.SeqToLabel.__init__ was configured with:\n",
      "{   'bidirectional': True,\n",
      "    'decode_dropout': 0.6,\n",
      "    'embedding_dropout': 0.3,\n",
      "    'embedding_size': 300,\n",
      "    'freeze_embeddings': True,\n",
      "    'input_vocab_size': 6693,\n",
      "    'output_vocab_size': 1702,\n",
      "    'rnn_cell': 'gru',\n",
      "    'rnn_dropout': 0.8,\n",
      "    'rnn_layers': 1,\n",
      "    'rnn_size': 450,\n",
      "    'rnn_variational_dropout': 0.0}\n",
      "[2018-02-02 13:31:38,579][MainProcess][lib.configurable][INFO] seq_encoder.SeqEncoder.__init__ no config for: lib.nn.seq_encoder.SeqEncoder.__init__\n",
      "[2018-02-02 13:31:38,579][MainProcess][lib.configurable][INFO] seq_encoder.SeqEncoder.__init__ was configured with:\n",
      "{   'bidirectional': True,\n",
      "    'embedding_dropout': 0.3,\n",
      "    'embedding_size': 300,\n",
      "    'freeze_embeddings': True,\n",
      "    'n_layers': 1,\n",
      "    'rnn_cell': 'gru',\n",
      "    'rnn_dropout': 0.8,\n",
      "    'rnn_size': 450,\n",
      "    'rnn_variational_dropout': 0.0,\n",
      "    'vocab_size': 6693}\n",
      "[2018-02-02 13:31:41,332][MainProcess][lib.configurable][INFO] adam.Adam.__init__ was configured with:\n",
      "{'amsgrad': True, 'params': <filter object at 0x7fdaf8527470>}\n",
      "[2018-02-02 13:31:41,334][MainProcess][lib.configurable][INFO] optimizer.Optimizer.__init__ no config for: lib.optimizer.Optimizer.__init__\n",
      "[2018-02-02 13:31:41,335][MainProcess][lib.configurable][INFO] optimizer.Optimizer.__init__ was configured with:\n",
      "{'optim': <lib.optim.adam.Adam object at 0x7fdaf8527438>}\n",
      "[2018-02-02 13:31:41,335][MainProcess][__main__][INFO] Epochs: 30\n",
      "[2018-02-02 13:31:41,336][MainProcess][__main__][INFO] Train Dataset Size: 75729\n",
      "[2018-02-02 13:31:41,336][MainProcess][__main__][INFO] Dev Dataset Size: 10816\n",
      "[2018-02-02 13:31:41,337][MainProcess][__main__][INFO] Train Batch Size: 32\n",
      "[2018-02-02 13:31:41,337][MainProcess][__main__][INFO] Train Max Batch Size: 1024\n",
      "[2018-02-02 13:31:41,338][MainProcess][__main__][INFO] Total Parameters: 1682902\n",
      "[2018-02-02 13:31:41,340][MainProcess][__main__][INFO] Model:\n",
      "SeqToLabel(\n",
      "  (encoder): SeqEncoder(\n",
      "    (embedding): Embedding(6693, 300, padding_idx=0)\n",
      "    (rnn): GRU(300, 225, bidirectional=True)\n",
      "    (rnn_dropout): LockedDropout(p=0.8)\n",
      "    (embedding_dropout): Dropout(p=0.3)\n",
      "  )\n",
      "  (out): Sequential(\n",
      "    (0): Linear(in_features=450, out_features=450)\n",
      "    (1): BatchNorm1d(450, eps=1e-05, momentum=0.1, affine=True)\n",
      "    (2): ReLU()\n",
      "    (3): Dropout(p=0.6)\n",
      "    (4): Linear(in_features=450, out_features=1702)\n",
      "  )\n",
      ")\n",
      "[2018-02-02 13:31:41,341][MainProcess][__main__][INFO] Epoch: 0\n",
      "[2018-02-02 13:31:41,341][MainProcess][__main__][INFO] Training...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0d2dddb2b4a7470b8bb0c301f2b16c4c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2367), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:31:52,224][MainProcess][lib.checkpoint][INFO] Saving checkpoint: ../../experiments/relation_classifier.02_02_13:31:11/13.pt\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b4e1873d9b9e44649c6e75ef4fd681e5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:31:52,784][MainProcess][__main__][INFO] NLLLoss: 0.942\n",
      "[2018-02-02 13:31:52,845][MainProcess][lib.metrics.accuracy][INFO] Accuracy: 0.8382951183431953 [9067 of 10816]\n",
      "––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––\n",
      "[2018-02-02 13:31:52,853][MainProcess][__main__][INFO] Epoch: 1\n",
      "[2018-02-02 13:31:52,854][MainProcess][__main__][INFO] Training...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f51c4f6bda874bfea6ed45be78d8dbb1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2367), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:32:03,128][MainProcess][lib.checkpoint][INFO] Saving checkpoint: ../../experiments/relation_classifier.02_02_13:31:11/24.pt\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fd3eba6af1de47ee862ba063876b4f6f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:32:03,513][MainProcess][__main__][INFO] NLLLoss: 0.785\n",
      "[2018-02-02 13:32:03,575][MainProcess][lib.metrics.accuracy][INFO] Accuracy: 0.8517011834319527 [9212 of 10816]\n",
      "––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––\n",
      "[2018-02-02 13:32:03,580][MainProcess][__main__][INFO] Epoch: 2\n",
      "[2018-02-02 13:32:03,581][MainProcess][__main__][INFO] Training...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "292686a816a64ddab37825096d19f4d2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2367), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:32:13,659][MainProcess][lib.checkpoint][INFO] Saving checkpoint: ../../experiments/relation_classifier.02_02_13:31:11/35.pt\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0a8a2babba494614bd00d3241b366b6e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:32:14,026][MainProcess][__main__][INFO] NLLLoss: 0.725\n",
      "[2018-02-02 13:32:14,087][MainProcess][lib.metrics.accuracy][INFO] Accuracy: 0.8638128698224852 [9343 of 10816]\n",
      "––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––\n",
      "[2018-02-02 13:32:14,092][MainProcess][__main__][INFO] Epoch: 3\n",
      "[2018-02-02 13:32:14,093][MainProcess][__main__][INFO] Training...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8c04280a913040a190598d9ee1124b02",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2367), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:32:24,340][MainProcess][lib.checkpoint][INFO] Saving checkpoint: ../../experiments/relation_classifier.02_02_13:31:11/45.pt\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e42b8b8965e9403b95a76a235c800570",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:32:24,851][MainProcess][__main__][INFO] NLLLoss: 0.694\n",
      "[2018-02-02 13:32:24,910][MainProcess][lib.metrics.accuracy][INFO] Accuracy: 0.8674186390532544 [9382 of 10816]\n",
      "––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––\n",
      "[2018-02-02 13:32:24,916][MainProcess][__main__][INFO] Epoch: 4\n",
      "[2018-02-02 13:32:24,917][MainProcess][__main__][INFO] Training...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6772b207689a44df803d7dd45bc7aa97",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2367), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:32:35,463][MainProcess][lib.checkpoint][INFO] Saving checkpoint: ../../experiments/relation_classifier.02_02_13:31:11/56.pt\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "59f5a2ea331141c58872f6d3519bf878",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:32:35,824][MainProcess][__main__][INFO] NLLLoss: 0.679\n",
      "[2018-02-02 13:32:35,886][MainProcess][lib.metrics.accuracy][INFO] Accuracy: 0.8750924556213018 [9465 of 10816]\n",
      "––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––\n",
      "[2018-02-02 13:32:35,891][MainProcess][__main__][INFO] Epoch: 5\n",
      "[2018-02-02 13:32:35,892][MainProcess][__main__][INFO] Training...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b73f6f67d63e482ea5fa6a558c80c741",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2367), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:32:46,827][MainProcess][lib.checkpoint][INFO] Saving checkpoint: ../../experiments/relation_classifier.02_02_13:31:11/68.pt\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2e8e34fbad4047ebaef733fa673c3b45",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:32:47,198][MainProcess][__main__][INFO] NLLLoss: 0.671\n",
      "[2018-02-02 13:32:47,259][MainProcess][lib.metrics.accuracy][INFO] Accuracy: 0.8761094674556213 [9476 of 10816]\n",
      "––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––\n",
      "[2018-02-02 13:32:47,264][MainProcess][__main__][INFO] Epoch: 6\n",
      "[2018-02-02 13:32:47,265][MainProcess][__main__][INFO] Training...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "942d851549994ecdb237e0496b240f15",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2367), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:32:57,486][MainProcess][lib.checkpoint][INFO] Saving checkpoint: ../../experiments/relation_classifier.02_02_13:31:11/78.pt\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6475309a3a844b7e9e3ecbb25f6e717d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:32:57,986][MainProcess][__main__][INFO] NLLLoss: 0.665\n",
      "[2018-02-02 13:32:58,044][MainProcess][lib.metrics.accuracy][INFO] Accuracy: 0.8776812130177515 [9493 of 10816]\n",
      "––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––\n",
      "[2018-02-02 13:32:58,050][MainProcess][__main__][INFO] Epoch: 7\n",
      "[2018-02-02 13:32:58,051][MainProcess][__main__][INFO] Training...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d84b1aa690b64849a8a2bb5aad40ddb1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2367), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:33:08,258][MainProcess][lib.checkpoint][INFO] Saving checkpoint: ../../experiments/relation_classifier.02_02_13:31:11/89.pt\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "887ab2a817de42ad8b8aac47a982f8e5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:33:08,630][MainProcess][__main__][INFO] NLLLoss: 0.667\n",
      "[2018-02-02 13:33:08,691][MainProcess][lib.metrics.accuracy][INFO] Accuracy: 0.8783284023668639 [9500 of 10816]\n",
      "––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––\n",
      "[2018-02-02 13:33:08,696][MainProcess][__main__][INFO] Epoch: 8\n",
      "[2018-02-02 13:33:08,697][MainProcess][__main__][INFO] Training...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f9585014402545b29cd65c52dca68a7f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2367), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:33:18,715][MainProcess][lib.checkpoint][INFO] Saving checkpoint: ../../experiments/relation_classifier.02_02_13:31:11/100.pt\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "def8c94cd7a441b9aac8adac071c93ad",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:33:19,080][MainProcess][__main__][INFO] NLLLoss: 0.662\n",
      "[2018-02-02 13:33:19,140][MainProcess][lib.metrics.accuracy][INFO] Accuracy: 0.8750924556213018 [9465 of 10816]\n",
      "––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––\n",
      "[2018-02-02 13:33:19,146][MainProcess][__main__][INFO] Epoch: 9\n",
      "[2018-02-02 13:33:19,146][MainProcess][__main__][INFO] Training...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5eecbe2ba33c4464a0ee37af9728850c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2367), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:33:29,173][MainProcess][lib.checkpoint][INFO] Saving checkpoint: ../../experiments/relation_classifier.02_02_13:31:11/110.pt\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ba251f4d121d425687c129c601363feb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:33:29,548][MainProcess][__main__][INFO] NLLLoss: 0.658\n",
      "[2018-02-02 13:33:29,608][MainProcess][lib.metrics.accuracy][INFO] Accuracy: 0.8777736686390533 [9494 of 10816]\n",
      "––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––\n",
      "[2018-02-02 13:33:29,613][MainProcess][__main__][INFO] Epoch: 10\n",
      "[2018-02-02 13:33:29,614][MainProcess][__main__][INFO] Training...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aff518de693c4399a9d3d2b56a45a31b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2367), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:33:39,922][MainProcess][lib.checkpoint][INFO] Saving checkpoint: ../../experiments/relation_classifier.02_02_13:31:11/121.pt\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "57f9ea3060eb4937b8190ab5cacbccb7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:33:40,313][MainProcess][__main__][INFO] NLLLoss: 0.667\n",
      "[2018-02-02 13:33:40,377][MainProcess][lib.metrics.accuracy][INFO] Accuracy: 0.8772189349112426 [9488 of 10816]\n",
      "[2018-02-02 13:33:40,382][MainProcess][__main__][INFO] Ran out of patience, increasing train batch size to: 64\n",
      "––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––\n",
      "[2018-02-02 13:33:40,383][MainProcess][__main__][INFO] Epoch: 11\n",
      "[2018-02-02 13:33:40,384][MainProcess][__main__][INFO] Training...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b8a99c6195294c3fa636ea8226a37e37",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1184), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:33:46,118][MainProcess][lib.checkpoint][INFO] Saving checkpoint: ../../experiments/relation_classifier.02_02_13:31:11/127.pt\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aaa6686a62974ef8bb6e0d8e19882d25",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:33:46,485][MainProcess][__main__][INFO] NLLLoss: 0.659\n",
      "[2018-02-02 13:33:46,550][MainProcess][lib.metrics.accuracy][INFO] Accuracy: 0.8806397928994083 [9525 of 10816]\n",
      "––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––\n",
      "[2018-02-02 13:33:46,556][MainProcess][__main__][INFO] Epoch: 12\n",
      "[2018-02-02 13:33:46,558][MainProcess][__main__][INFO] Training...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f90ff34801fc4a12af797555cd1d6839",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1184), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:33:52,484][MainProcess][lib.checkpoint][INFO] Saving checkpoint: ../../experiments/relation_classifier.02_02_13:31:11/133.pt\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "73dde5d1ca774e99b8abd07c15fc8e8b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:33:53,082][MainProcess][__main__][INFO] NLLLoss: 0.657\n",
      "[2018-02-02 13:33:53,147][MainProcess][lib.metrics.accuracy][INFO] Accuracy: 0.8820266272189349 [9540 of 10816]\n",
      "––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––\n",
      "[2018-02-02 13:33:53,153][MainProcess][__main__][INFO] Epoch: 13\n",
      "[2018-02-02 13:33:53,154][MainProcess][__main__][INFO] Training...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f8744625b89b416db0c8ceb14ecf2630",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1184), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:33:59,211][MainProcess][lib.checkpoint][INFO] Saving checkpoint: ../../experiments/relation_classifier.02_02_13:31:11/140.pt\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3ab2835c36774c12b1bccf507f9e52e2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:33:59,614][MainProcess][__main__][INFO] NLLLoss: 0.664\n",
      "[2018-02-02 13:33:59,680][MainProcess][lib.metrics.accuracy][INFO] Accuracy: 0.8819341715976331 [9539 of 10816]\n",
      "––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––\n",
      "[2018-02-02 13:33:59,688][MainProcess][__main__][INFO] Epoch: 14\n",
      "[2018-02-02 13:33:59,688][MainProcess][__main__][INFO] Training...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "385f6ef69ff04f7893eba7c24119e455",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1184), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:34:06,086][MainProcess][lib.checkpoint][INFO] Saving checkpoint: ../../experiments/relation_classifier.02_02_13:31:11/147.pt\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "be57924755214111aa50db6bbb8e8d25",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:34:06,481][MainProcess][__main__][INFO] NLLLoss: 0.668\n",
      "[2018-02-02 13:34:06,545][MainProcess][lib.metrics.accuracy][INFO] Accuracy: 0.8814718934911243 [9534 of 10816]\n",
      "––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––\n",
      "[2018-02-02 13:34:06,551][MainProcess][__main__][INFO] Epoch: 15\n",
      "[2018-02-02 13:34:06,552][MainProcess][__main__][INFO] Training...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "90e6621ad4a9429981b026ddcf9b07d0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1184), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:34:12,921][MainProcess][lib.checkpoint][INFO] Saving checkpoint: ../../experiments/relation_classifier.02_02_13:31:11/154.pt\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d7cc6676c37a4471abc6bb04545bab0f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:34:13,449][MainProcess][__main__][INFO] NLLLoss: 0.669\n",
      "[2018-02-02 13:34:13,511][MainProcess][lib.metrics.accuracy][INFO] Accuracy: 0.8812869822485208 [9532 of 10816]\n",
      "[2018-02-02 13:34:13,518][MainProcess][__main__][INFO] Ran out of patience, increasing train batch size to: 128\n",
      "––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––\n",
      "[2018-02-02 13:34:13,519][MainProcess][__main__][INFO] Epoch: 16\n",
      "[2018-02-02 13:34:13,519][MainProcess][__main__][INFO] Training...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9ccc505707064daba036de301fb115a1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=592), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:34:16,947][MainProcess][lib.checkpoint][INFO] Saving checkpoint: ../../experiments/relation_classifier.02_02_13:31:11/158.pt\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e5a7066f4c0c44c1bb91d586869152db",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:34:17,307][MainProcess][__main__][INFO] NLLLoss: 0.670\n",
      "[2018-02-02 13:34:17,368][MainProcess][lib.metrics.accuracy][INFO] Accuracy: 0.8812869822485208 [9532 of 10816]\n",
      "[2018-02-02 13:34:17,373][MainProcess][__main__][INFO] Ran out of patience, increasing train batch size to: 256\n",
      "––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––\n",
      "[2018-02-02 13:34:17,374][MainProcess][__main__][INFO] Epoch: 17\n",
      "[2018-02-02 13:34:17,376][MainProcess][__main__][INFO] Training...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f7129ff0eebc46408d990a453dc43d9a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=296), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:34:19,610][MainProcess][lib.checkpoint][INFO] Saving checkpoint: ../../experiments/relation_classifier.02_02_13:31:11/161.pt\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c14dd34b71094491b8268bf6017358c8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:34:20,007][MainProcess][__main__][INFO] NLLLoss: 0.671\n",
      "[2018-02-02 13:34:20,073][MainProcess][lib.metrics.accuracy][INFO] Accuracy: 0.8830436390532544 [9551 of 10816]\n",
      "––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––\n",
      "[2018-02-02 13:34:20,078][MainProcess][__main__][INFO] Epoch: 18\n",
      "[2018-02-02 13:34:20,079][MainProcess][__main__][INFO] Training...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "11d7a696a623403c95df4f2f59d66671",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=296), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:34:22,299][MainProcess][lib.checkpoint][INFO] Saving checkpoint: ../../experiments/relation_classifier.02_02_13:31:11/163.pt\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f91408fe23fc4cb1a8fe736eb2f60322",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:34:22,842][MainProcess][__main__][INFO] NLLLoss: 0.671\n",
      "[2018-02-02 13:34:22,903][MainProcess][lib.metrics.accuracy][INFO] Accuracy: 0.882396449704142 [9544 of 10816]\n",
      "––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––\n",
      "[2018-02-02 13:34:22,909][MainProcess][__main__][INFO] Epoch: 19\n",
      "[2018-02-02 13:34:22,910][MainProcess][__main__][INFO] Training...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2f40a29c99ce4da6bfacd4ec852e510d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=296), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:34:25,123][MainProcess][lib.checkpoint][INFO] Saving checkpoint: ../../experiments/relation_classifier.02_02_13:31:11/166.pt\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "03bdb1bbe7dc46c792a6397f2268c784",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:34:25,511][MainProcess][__main__][INFO] NLLLoss: 0.673\n",
      "[2018-02-02 13:34:25,574][MainProcess][lib.metrics.accuracy][INFO] Accuracy: 0.8813794378698225 [9533 of 10816]\n",
      "––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––\n",
      "[2018-02-02 13:34:25,579][MainProcess][__main__][INFO] Epoch: 20\n",
      "[2018-02-02 13:34:25,580][MainProcess][__main__][INFO] Training...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b9931990f19342868967c92ba0a20e5a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=296), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:34:27,820][MainProcess][lib.checkpoint][INFO] Saving checkpoint: ../../experiments/relation_classifier.02_02_13:31:11/169.pt\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3f8af35b9bb14ef5b5066043e759f7a8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:34:28,207][MainProcess][__main__][INFO] NLLLoss: 0.673\n",
      "[2018-02-02 13:34:28,276][MainProcess][lib.metrics.accuracy][INFO] Accuracy: 0.8833210059171598 [9554 of 10816]\n",
      "––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––\n",
      "[2018-02-02 13:34:28,283][MainProcess][__main__][INFO] Epoch: 21\n",
      "[2018-02-02 13:34:28,284][MainProcess][__main__][INFO] Training...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "50491fdef2e74eb996fb9facf5790c2d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=296), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:34:30,500][MainProcess][lib.checkpoint][INFO] Saving checkpoint: ../../experiments/relation_classifier.02_02_13:31:11/171.pt\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8f9de7d8e74d4028a78cc37f4a1b70d0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:34:30,868][MainProcess][__main__][INFO] NLLLoss: 0.679\n",
      "[2018-02-02 13:34:30,929][MainProcess][lib.metrics.accuracy][INFO] Accuracy: 0.8828587278106509 [9549 of 10816]\n",
      "––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––\n",
      "[2018-02-02 13:34:30,934][MainProcess][__main__][INFO] Epoch: 22\n",
      "[2018-02-02 13:34:30,935][MainProcess][__main__][INFO] Training...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "053edbc17f1e4653bfe3a0e0d170bfa5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=296), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:34:33,170][MainProcess][lib.checkpoint][INFO] Saving checkpoint: ../../experiments/relation_classifier.02_02_13:31:11/174.pt\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "75ac8687adfd4f3197971f3e13b4bdbb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:34:33,718][MainProcess][__main__][INFO] NLLLoss: 0.680\n",
      "[2018-02-02 13:34:33,779][MainProcess][lib.metrics.accuracy][INFO] Accuracy: 0.8835983727810651 [9557 of 10816]\n",
      "––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––\n",
      "[2018-02-02 13:34:33,784][MainProcess][__main__][INFO] Epoch: 23\n",
      "[2018-02-02 13:34:33,785][MainProcess][__main__][INFO] Training...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ef63e59598a4417f8fa0c006c46b0b3e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=296), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:34:36,010][MainProcess][lib.checkpoint][INFO] Saving checkpoint: ../../experiments/relation_classifier.02_02_13:31:11/177.pt\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "00ab8779a8914926919badea6ae37b59",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:34:36,408][MainProcess][__main__][INFO] NLLLoss: 0.684\n",
      "[2018-02-02 13:34:36,469][MainProcess][lib.metrics.accuracy][INFO] Accuracy: 0.8825813609467456 [9546 of 10816]\n",
      "––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––\n",
      "[2018-02-02 13:34:36,475][MainProcess][__main__][INFO] Epoch: 24\n",
      "[2018-02-02 13:34:36,476][MainProcess][__main__][INFO] Training...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a6babe8d274f492b87391047283a95e6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=296), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:34:38,830][MainProcess][lib.checkpoint][INFO] Saving checkpoint: ../../experiments/relation_classifier.02_02_13:31:11/180.pt\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bc67033444e540dfb6c4358f3530245f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:34:39,250][MainProcess][__main__][INFO] NLLLoss: 0.684\n",
      "[2018-02-02 13:34:39,311][MainProcess][lib.metrics.accuracy][INFO] Accuracy: 0.8828587278106509 [9549 of 10816]\n",
      "––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––\n",
      "[2018-02-02 13:34:39,317][MainProcess][__main__][INFO] Epoch: 25\n",
      "[2018-02-02 13:34:39,318][MainProcess][__main__][INFO] Training...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d34d15495143449b98b337d7351d0e34",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=296), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:34:41,796][MainProcess][lib.checkpoint][INFO] Saving checkpoint: ../../experiments/relation_classifier.02_02_13:31:11/183.pt\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fc56be1fae42490caf50c2fb62b2916f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:34:42,309][MainProcess][__main__][INFO] NLLLoss: 0.683\n",
      "[2018-02-02 13:34:42,375][MainProcess][lib.metrics.accuracy][INFO] Accuracy: 0.8833210059171598 [9554 of 10816]\n",
      "[2018-02-02 13:34:42,381][MainProcess][__main__][INFO] Ran out of patience, increasing train batch size to: 512\n",
      "––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––\n",
      "[2018-02-02 13:34:42,382][MainProcess][__main__][INFO] Epoch: 26\n",
      "[2018-02-02 13:34:42,383][MainProcess][__main__][INFO] Training...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2589a493677e45a8b80e5ea5ed9af1ef",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=148), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:34:44,110][MainProcess][lib.checkpoint][INFO] Saving checkpoint: ../../experiments/relation_classifier.02_02_13:31:11/185.pt\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7a2b55a7d751490f85e7551569803d72",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:34:44,481][MainProcess][__main__][INFO] NLLLoss: 0.684\n",
      "[2018-02-02 13:34:44,542][MainProcess][lib.metrics.accuracy][INFO] Accuracy: 0.8835059171597633 [9556 of 10816]\n",
      "[2018-02-02 13:34:44,548][MainProcess][__main__][INFO] Ran out of patience, increasing train batch size to: 1024\n",
      "––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––\n",
      "[2018-02-02 13:34:44,549][MainProcess][__main__][INFO] Epoch: 27\n",
      "[2018-02-02 13:34:44,550][MainProcess][__main__][INFO] Training...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f0fa662e3bb84ae8a9a31e16e22f8e69",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=74), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:34:46,199][MainProcess][lib.checkpoint][INFO] Saving checkpoint: ../../experiments/relation_classifier.02_02_13:31:11/187.pt\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aa51f935ca6240e7ab851222349f7f0e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:34:46,568][MainProcess][__main__][INFO] NLLLoss: 0.685\n",
      "[2018-02-02 13:34:46,634][MainProcess][lib.metrics.accuracy][INFO] Accuracy: 0.8833210059171598 [9554 of 10816]\n",
      "[2018-02-02 13:34:46,639][MainProcess][__main__][INFO] Ran out of patience, increasing train batch size to: 1024\n",
      "––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––\n",
      "[2018-02-02 13:34:46,642][MainProcess][__main__][INFO] Epoch: 28\n",
      "[2018-02-02 13:34:46,643][MainProcess][__main__][INFO] Training...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ec7834e5b19a4af5ab3b3968d8590ce2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=74), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:34:48,117][MainProcess][lib.checkpoint][INFO] Saving checkpoint: ../../experiments/relation_classifier.02_02_13:31:11/189.pt\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2201feda9b634111b9f5eaf60299def5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:34:48,483][MainProcess][__main__][INFO] NLLLoss: 0.684\n",
      "[2018-02-02 13:34:48,545][MainProcess][lib.metrics.accuracy][INFO] Accuracy: 0.8837832840236687 [9559 of 10816]\n",
      "––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––\n",
      "[2018-02-02 13:34:48,551][MainProcess][__main__][INFO] Epoch: 29\n",
      "[2018-02-02 13:34:48,552][MainProcess][__main__][INFO] Training...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7acf2c5f5bdb41f4affccbbbbf43fdf3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=74), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:34:50,262][MainProcess][lib.checkpoint][INFO] Saving checkpoint: ../../experiments/relation_classifier.02_02_13:31:11/191.pt\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9549e86c492d4f67a4a15fc8308981fd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[2018-02-02 13:34:50,631][MainProcess][__main__][INFO] NLLLoss: 0.686\n",
      "[2018-02-02 13:34:50,692][MainProcess][lib.metrics.accuracy][INFO] Accuracy: 0.8830436390532544 [9551 of 10816]\n",
      "––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––––\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(-0.8837832840236687,\n",
       " '../../experiments/relation_classifier.02_02_13:31:11/191.pt')"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hyperparameter Optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import random\n",
    "\n",
    "from skopt.space import Real, Integer, Categorical\n",
    "\n",
    "from lib.hyperparameter_optimization import hyperband\n",
    "from lib.configurable import add_config\n",
    "from lib.configurable import log_config\n",
    "\n",
    "space  = [Real(0, 0.9, name='embedding_dropout'),\n",
    "          Real(0, 0.9, name='rnn_variational_dropout'),\n",
    "          Real(0, 0.9, name='rnn_dropout'),\n",
    "          Real(0, 0.9, name='decode_dropout'),\n",
    "          Integer(50, 250, name='rnn_size'), # We multiply this hyperparameter by two to allow for bidirectional\n",
    "          Integer(1, 3, name='rnn_layers'),\n",
    "          Categorical(['lstm', 'gru'], name='rnn_cell'),\n",
    "          Categorical([True, False], name='freeze_embeddings'),\n",
    "          Categorical([True, False], name='bidirectional'),\n",
    "          Real(0, 5, name='max_grad_norm')]\n",
    "\n",
    "def objective(embedding_dropout, rnn_variational_dropout, rnn_dropout, decode_dropout,\n",
    "              rnn_size, rnn_layers, rnn_cell, freeze_embeddings, bidirectional, max_grad_norm,\n",
    "              *args, **kwargs):\n",
    "\n",
    "    add_config({\n",
    "        'lib': {\n",
    "            'nn.seq_to_label.SeqToLabel.__init__': {\n",
    "                'bidirectional': bidirectional,\n",
    "                'embedding_size': 300,\n",
    "                'rnn_size': int(rnn_size) * 2,\n",
    "                'freeze_embeddings': freeze_embeddings,\n",
    "                'rnn_cell': rnn_cell,\n",
    "                'decode_dropout': decode_dropout,  # dropout before fully connected layer in RNN\n",
    "                'rnn_layers': int(rnn_layers),\n",
    "                'rnn_variational_dropout': rnn_variational_dropout,\n",
    "                'embedding_dropout': embedding_dropout,\n",
    "                'rnn_dropout': rnn_dropout,\n",
    "            },\n",
    "            'optimizer.Optimizer.__init__.max_grad_norm': max_grad_norm,\n",
    "        },\n",
    "        'lib.optim.adam.Adam.__init__': {\n",
    "            'amsgrad': True\n",
    "        }\n",
    "    })\n",
    "\n",
    "    ret = train(*args, **kwargs)\n",
    "    print('=' * 100)\n",
    "    torch.cuda.empty_cache()\n",
    "    return ret\n",
    "\n",
    "scores, hyperparameters = hyperband(objective, space, max_resources_per_model=30, total_resources=1000)\n",
    "print('Best Accuracy: %.4f' % min(scores))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}