01_NER-Notebook.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\felix\\Anaconda3\\lib\\site-packages\\tqdm\\autonotebook\\__init__.py:14: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n",
      "  \" (e.g. in jupyter console)\", TqdmExperimentalWarning)\n"
     ]
    }
   ],
   "source": [
    "from data.conll import conll2003_dataset, extract_samples\n",
    "from misc.preferences import PREFERENCES\n",
    "from misc.visualizer import *\n",
    "from misc.hyperparameters import get_default_params\n",
    "from optimizer import get_default_optimizer\n",
    "from misc import utils\n",
    "from models.transformer.encoder import TransformerEncoder\n",
    "from models.softmax_output import SoftmaxOutputLayer, OutputLayer\n",
    "from models.transformer_tagger import TransformerTagger\n",
    "from models.transformer.train import Trainer\n",
    "from criterion import NllLoss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "experiment_name = 'just_testing'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log path is  C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\just_testing\n"
     ]
    }
   ],
   "source": [
    "PREFERENCES.defaults(\n",
    "    data_root='./data/conll2003',\n",
    "    data_train='eng.train.txt',\n",
    "    data_validation='eng.testa.txt',\n",
    "    data_test='eng.testb.txt',\n",
    "    early_stopping='highest_5_F1'\n",
    ")\n",
    "\n",
    "hyper_parameters = get_default_params()\n",
    "hyper_parameters.model_size = 300\n",
    "hyper_parameters.batch_size = 80\n",
    "hyper_parameters.early_stopping = -1\n",
    "experiment_name = utils.create_loggers(experiment_name=experiment_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "conll2003 = conll2003_dataset('ner', hyper_parameters.batch_size,\n",
    "                              root=PREFERENCES.data_root,\n",
    "                              train_file=PREFERENCES.data_train,\n",
    "                              validation_file=PREFERENCES.data_validation,\n",
    "                              test_file=PREFERENCES.data_test,\n",
    "                              use_cuda=True)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "eu - I-ORG\n",
      "rejects - O\n",
      "german - I-MISC\n",
      "call - O\n",
      "to - O\n",
      "boycott - O\n",
      "british - I-MISC\n",
      "lamb - O\n",
      ". - O\n",
      "\n",
      "#######################\n",
      "\n",
      "peter - I-PER\n",
      "blackburn - I-PER\n",
      "\n",
      "#######################\n",
      "\n",
      "brussels - I-LOC\n",
      "1996-08-22 - O\n",
      "\n",
      "#######################\n",
      "\n",
      "cricket - O\n",
      "- - O\n",
      "leicestershire - I-ORG\n",
      "take - O\n",
      "over - O\n",
      "at - O\n",
      "top - O\n",
      "after - O\n",
      "innings - O\n",
      "victory - O\n",
      ". - O\n",
      "\n",
      "#######################\n",
      "\n",
      "london - I-LOC\n",
      "1996-08-30 - O\n",
      "\n",
      "#######################\n",
      "\n",
      "west - I-MISC\n",
      "indian - I-MISC\n",
      "all-rounder - O\n",
      "phil - I-PER\n",
      "simmons - I-PER\n",
      "took - O\n",
      "four - O\n",
      "for - O\n",
      "0 - O\n",
      "on - O\n",
      "friday - O\n",
      "as - O\n",
      "leicestershire - I-ORG\n",
      "beat - O\n",
      "somerset - I-ORG\n",
      "by - O\n",
      "an - O\n",
      "innings - O\n",
      "and - O\n",
      "0 - O\n",
      "runs - O\n",
      "in - O\n",
      "two - O\n",
      "days - O\n",
      "to - O\n",
      "take - O\n",
      "over - O\n",
      "at - O\n",
      "the - O\n",
      "head - O\n",
      "of - O\n",
      "the - O\n",
      "county - O\n",
      "championship - O\n",
      ". - O\n",
      "\n",
      "#######################\n",
      "\n",
      "soccer - O\n",
      "- - O\n",
      "japan - I-LOC\n",
      "get - O\n",
      "lucky - O\n",
      "win - O\n",
      ", - O\n",
      "china - I-PER\n",
      "in - O\n",
      "surprise - O\n",
      "defeat - O\n",
      ". - O\n",
      "\n",
      "#######################\n",
      "\n",
      "nadim - I-PER\n",
      "ladki - I-PER\n",
      "\n",
      "#######################\n",
      "\n",
      "al-ain - I-LOC\n",
      ", - O\n",
      "united - I-LOC\n",
      "arab - I-LOC\n",
      "emirates - I-LOC\n",
      "1996-12-06 - O\n",
      "\n",
      "#######################\n",
      "\n"
     ]
    }
   ],
   "source": [
    "samples = extract_samples(conll2003['examples'])\n",
    "print_samples(samples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Targets: dict_keys(['I-ORG', 'O', 'I-MISC', 'I-PER', 'I-LOC', 'B-LOC', 'B-MISC', 'B-ORG'])\n",
      "Counter({'O': 169578, 'I-PER': 11128, 'I-ORG': 10001, 'I-LOC': 8286, 'I-MISC': 4556, 'B-MISC': 37, 'B-ORG': 24, 'B-LOC': 11})\n",
      "[' UNK ', '<pad>', 'O', 'I-PER', 'I-ORG', 'I-LOC', 'I-MISC', 'B-MISC', 'B-ORG', 'B-LOC']\n"
     ]
    }
   ],
   "source": [
    "# 10 words with a 100-length embedding\n",
    "target_vocab = conll2003['vocabs'][1]\n",
    "target_size = len(target_vocab)\n",
    "print('Targets:',target_vocab.freqs.keys())\n",
    "print(target_vocab.freqs)\n",
    "print(target_vocab.itos)\n",
    "class_labels = target_vocab.itos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I-ORG: 10001\n",
      "O: 169578\n",
      "I-MISC: 4556\n",
      "I-PER: 11128\n",
      "I-LOC: 8286\n",
      "B-LOC: 11\n",
      "B-MISC: 37\n",
      "B-ORG: 24\n",
      "\n",
      "Total Samples: 203621\n",
      "\n",
      "\n",
      "Trivial classifiers\n",
      "Trivial I-ORG: 4.9115759180045275%\n",
      "Trivial O: 83.2811939829389%\n",
      "Trivial I-MISC: 2.2374902392189413%\n",
      "Trivial I-PER: 5.4650551760378345%\n",
      "Trivial I-LOC: 4.069324873171235%\n",
      "Trivial B-LOC: 0.005402193290475934%\n",
      "Trivial B-MISC: 0.01817101379523723%\n",
      "Trivial B-ORG: 0.011786603542856582%\n"
     ]
    }
   ],
   "source": [
    "total_samples = 0\n",
    "for l, freq in target_vocab.freqs.items():\n",
    "    print('{}: {}'.format(l, freq))\n",
    "    total_samples += freq\n",
    "print('\\nTotal Samples:',total_samples)\n",
    "\n",
    "print('\\n\\nTrivial classifiers')\n",
    "for l, freq in target_vocab.freqs.items():\n",
    "    acc = float(freq) / float(total_samples)\n",
    "    print('Trivial {}: {}%'.format(l, acc*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "loss = NllLoss(target_size)\n",
    "# transformer = GoogleTransformer(True, target_size, target_size, num_units, 2, 2, 512, 0.1)\n",
    "transformer = TransformerEncoder(conll2003['embeddings'][0],\n",
    "                                 n_enc_blocks=2,\n",
    "                                 n_head=3,\n",
    "                                 d_model=hyper_parameters.model_size,\n",
    "                                 d_k=100,\n",
    "                                 d_v=100)\n",
    "tagging_softmax = SoftmaxOutputLayer(hyper_parameters.model_size, target_size)\n",
    "model = TransformerTagger(transformer, tagging_softmax)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# predict now to see model in final state\n",
    "#df = predict_some_examples_to_df(model, conll2003['iters'][1], num_samples=400)\n",
    "#print(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------------------------\n",
      "        Layer (type)               Output Shape         Param #\n",
      "================================================================\n",
      "         Embedding-1              [-1, 42, 300]       7,816,800\n",
      "           Dropout-2              [-1, 42, 300]               0\n",
      "PositionalEncoding2-3              [-1, 42, 300]               0\n",
      "            Linear-4              [-1, 42, 300]          90,000\n",
      "            Linear-5              [-1, 42, 300]          90,000\n",
      "            Linear-6              [-1, 42, 300]          90,000\n",
      "           Dropout-7               [-1, 42, 42]               0\n",
      "ScaledDotProductAttentionLayer-8              [-1, 42, 100]               0\n",
      "            Linear-9              [-1, 42, 300]          90,000\n",
      "          Dropout-10              [-1, 42, 300]               0\n",
      "        LayerNorm-11              [-1, 42, 300]               0\n",
      "MultiHeadedSelfAttentionLayer-12              [-1, 42, 300]               0\n",
      "           Linear-13             [-1, 42, 2048]         616,448\n",
      "           Linear-14              [-1, 42, 300]         614,700\n",
      "          Dropout-15              [-1, 42, 300]               0\n",
      "        LayerNorm-16              [-1, 42, 300]               0\n",
      "     EncoderBlock-17              [-1, 42, 300]               0\n",
      "           Linear-18              [-1, 42, 300]          90,000\n",
      "           Linear-19              [-1, 42, 300]          90,000\n",
      "           Linear-20              [-1, 42, 300]          90,000\n",
      "          Dropout-21               [-1, 42, 42]               0\n",
      "ScaledDotProductAttentionLayer-22              [-1, 42, 100]               0\n",
      "           Linear-23              [-1, 42, 300]          90,000\n",
      "          Dropout-24              [-1, 42, 300]               0\n",
      "        LayerNorm-25              [-1, 42, 300]               0\n",
      "MultiHeadedSelfAttentionLayer-26              [-1, 42, 300]               0\n",
      "           Linear-27             [-1, 42, 2048]         616,448\n",
      "           Linear-28              [-1, 42, 300]         614,700\n",
      "          Dropout-29              [-1, 42, 300]               0\n",
      "        LayerNorm-30              [-1, 42, 300]               0\n",
      "     EncoderBlock-31              [-1, 42, 300]               0\n",
      "TransformerEncoder-32              [-1, 42, 300]               0\n",
      "           Linear-33               [-1, 42, 10]           3,010\n",
      "SoftmaxOutputLayer-34               [-1, 42, 10]               0\n",
      "================================================================\n",
      "Total params: 11,002,106\n",
      "Trainable params: 11,002,106\n",
      "Non-trainable params: 0\n",
      "----------------------------------------------------------------\n",
      "Input size (MB): 0.00\n",
      "Forward/backward pass size (MB): 3.91\n",
      "Params size (MB): 41.97\n",
      "Estimated Total Size (MB): 45.88\n",
      "----------------------------------------------------------------\n",
      "pre_training - INFO - None\n",
      "pre_training - INFO - Classes: [' UNK ', '<pad>', 'O', 'I-PER', 'I-ORG', 'I-LOC', 'I-MISC', 'B-MISC', 'B-ORG', 'B-LOC']\n",
      "pre_training - INFO - TransformerTagger (\n",
      "  (encoder): TransformerEncoder(\n",
      "    (src_embeddings): Embedding(26056, 300)\n",
      "    (positional_encoding): PositionalEncoding2(\n",
      "      (dropout): Dropout(p=0.1)\n",
      "    )\n",
      "    (encoder_blocks): ModuleList(\n",
      "      (0): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.1)\n",
      "          )\n",
      "          (layer_norm): LayerNorm()\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.1)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (layer_norm): LayerNorm()\n",
      "          (w_1): Linear(in_features=300, out_features=2048, bias=True)\n",
      "          (w_2): Linear(in_features=2048, out_features=300, bias=True)\n",
      "          (dropout): Dropout(p=0.1)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "      (1): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.1)\n",
      "          )\n",
      "          (layer_norm): LayerNorm()\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.1)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (layer_norm): LayerNorm()\n",
      "          (w_1): Linear(in_features=300, out_features=2048, bias=True)\n",
      "          (w_2): Linear(in_features=2048, out_features=300, bias=True)\n",
      "          (dropout): Dropout(p=0.1)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "    )\n",
      "    (layer_norm): LayerNorm()\n",
      "  ), weights=((26056, 300), (300, 300), (300, 300), (300, 300), (300,), (300,), (300, 300), (300,), (300,), (2048, 300), (2048,), (300, 2048), (300,), (300,), (300,), (300, 300), (300, 300), (300, 300), (300,), (300,), (300, 300), (300,), (300,), (2048, 300), (2048,), (300, 2048), (300,), (300,), (300,), (300,), (300,)), parameters=11003296\n",
      "  (taggingLayer): SoftmaxOutputLayer(\n",
      "    (output_projection): Linear(in_features=300, out_features=10, bias=True)\n",
      "  ), weights=((10, 300), (10,)), parameters=3010\n",
      ")\n",
      "==================================\n",
      "Total Number of parameters: 11.006.306\n",
      "==================================\n",
      "\n",
      "graph(%input.1 : Long(80, 42)\n",
      "      %1 : Float(26056, 300)\n",
      "      %2 : Float(1, 200, 300)\n",
      "      %3 : Float(300, 300)\n",
      "      %4 : Float(300, 300)\n",
      "      %5 : Float(300, 300)\n",
      "      %6 : Float(300)\n",
      "      %7 : Float(300)\n",
      "      %8 : Float(300, 300)\n",
      "      %9 : Float(300)\n",
      "      %10 : Float(300)\n",
      "      %11 : Float(2048, 300)\n",
      "      %12 : Float(2048)\n",
      "      %13 : Float(300, 2048)\n",
      "      %14 : Float(300)\n",
      "      %15 : Float(300)\n",
      "      %16 : Float(300)\n",
      "      %17 : Float(300, 300)\n",
      "      %18 : Float(300, 300)\n",
      "      %19 : Float(300, 300)\n",
      "      %20 : Float(300)\n",
      "      %21 : Float(300)\n",
      "      %22 : Float(300, 300)\n",
      "      %23 : Float(300)\n",
      "      %24 : Float(300)\n",
      "      %25 : Float(2048, 300)\n",
      "      %26 : Float(2048)\n",
      "      %27 : Float(300, 2048)\n",
      "      %28 : Float(300)\n",
      "      %29 : Float(300)\n",
      "      %30 : Float(300)\n",
      "      %31 : Float(300)\n",
      "      %32 : Float(300)\n",
      "      %33 : Float(10, 300)\n",
      "      %34 : Float(10)) {\n",
      "  %35 : Float(80, 42, 300) = onnx::Gather(%1, %input.1), scope: TransformerTagger/TransformerEncoder[encoder]/Embedding[src_embeddings]\n",
      "  %36 : Tensor = onnx::Constant[value={17.3205}](), scope: TransformerTagger/TransformerEncoder[encoder]/PositionalEncoding2[positional_encoding]\n",
      "  %37 : Float(80, 42, 300) = onnx::Mul(%35, %36), scope: TransformerTagger/TransformerEncoder[encoder]/PositionalEncoding2[positional_encoding]\n",
      "  %38 : Float(1, 200, 300) = onnx::Slice[axes=[0], ends=[9223372036854775807], starts=[0]](%2), scope: TransformerTagger/TransformerEncoder[encoder]/PositionalEncoding2[positional_encoding]\n",
      "  %39 : Float(1!, 42, 300) = onnx::Slice[axes=[1], ends=[42], starts=[0]](%38), scope: TransformerTagger/TransformerEncoder[encoder]/PositionalEncoding2[positional_encoding]\n",
      "  %40 : Float(80, 42, 300) = onnx::Add(%37, %39), scope: TransformerTagger/TransformerEncoder[encoder]/PositionalEncoding2[positional_encoding]\n",
      "  %41 : Float(80, 42, 300), %42 : Tensor = onnx::Dropout[ratio=0.1](%40), scope: TransformerTagger/TransformerEncoder[encoder]/PositionalEncoding2[positional_encoding]/Dropout[dropout]\n",
      "  %43 : Float(300!, 300!) = onnx::Transpose[perm=[1, 0]](%3), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/Linear[query_projections]\n",
      "  %44 : Float(80, 42, 300) = onnx::MatMul(%41, %43), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/Linear[query_projections]\n",
      "  %45 : Float(300!, 300!) = onnx::Transpose[perm=[1, 0]](%4), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/Linear[key_projections]\n",
      "  %46 : Float(80, 42, 300) = onnx::MatMul(%41, %45), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/Linear[key_projections]\n",
      "  %47 : Float(300!, 300!) = onnx::Transpose[perm=[1, 0]](%5), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/Linear[value_projections]\n",
      "  %48 : Float(80, 42, 300) = onnx::MatMul(%41, %47), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/Linear[value_projections]\n",
      "  %49 : Long() = onnx::Constant[value={0}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %50 : Tensor = onnx::Shape(%41), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %51 : Long() = onnx::Gather[axis=0](%50, %49), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %52 : Long() = onnx::Constant[value={1}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %53 : Tensor = onnx::Shape(%41), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %54 : Long() = onnx::Gather[axis=0](%53, %52), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %55 : Long() = onnx::Constant[value={1}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %56 : Tensor = onnx::Shape(%41), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %57 : Long() = onnx::Gather[axis=0](%56, %55), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %58 : Long() = onnx::Constant[value={1}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %59 : Tensor = onnx::Shape(%41), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %60 : Long() = onnx::Gather[axis=0](%59, %58), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %61 : Long() = onnx::Constant[value={3}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %62 : Long() = onnx::Constant[value={100}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %63 : int[] = prim::ListConstruct(%51, %54, %61, %62), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %64 : Float(80, 42, 3, 100) = onnx::Reshape(%44, %63), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %65 : Long() = onnx::Constant[value={3}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %66 : Long() = onnx::Constant[value={100}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %67 : int[] = prim::ListConstruct(%51, %57, %65, %66), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %68 : Float(80, 42, 3, 100) = onnx::Reshape(%46, %67), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %69 : Long() = onnx::Constant[value={3}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %70 : Long() = onnx::Constant[value={100}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %71 : int[] = prim::ListConstruct(%51, %60, %69, %70), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %72 : Float(80, 42, 3, 100) = onnx::Reshape(%48, %71), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %73 : Float(3, 80, 42, 100) = onnx::Transpose[perm=[2, 0, 1, 3]](%64), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %74 : Long() = onnx::Constant[value={-1}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %75 : Long() = onnx::Constant[value={100}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %76 : int[] = prim::ListConstruct(%74, %54, %75), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %77 : Float(240, 42, 100) = onnx::Reshape(%73, %76), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %78 : Float(3, 80, 42, 100) = onnx::Transpose[perm=[2, 0, 1, 3]](%68), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %79 : Long() = onnx::Constant[value={-1}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %80 : Long() = onnx::Constant[value={100}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %81 : int[] = prim::ListConstruct(%79, %57, %80), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %82 : Float(240, 42, 100) = onnx::Reshape(%78, %81), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %83 : Float(3, 80, 42, 100) = onnx::Transpose[perm=[2, 0, 1, 3]](%72), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %84 : Long() = onnx::Constant[value={-1}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %85 : Long() = onnx::Constant[value={100}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %86 : int[] = prim::ListConstruct(%84, %60, %85), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %87 : Float(240, 42, 100) = onnx::Reshape(%83, %86), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %88 : Float(240!, 100!, 42!) = onnx::Transpose[perm=[0, 2, 1]](%82), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/ScaledDotProductAttentionLayer[attention_layer]\n",
      "  %89 : Float(240, 42, 42) = onnx::MatMul(%77, %88), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/ScaledDotProductAttentionLayer[attention_layer]\n",
      "  %90 : Tensor = onnx::Constant[value={8}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/ScaledDotProductAttentionLayer[attention_layer]\n",
      "  %91 : Float(240, 42, 42) = onnx::Div(%89, %90), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/ScaledDotProductAttentionLayer[attention_layer]\n",
      "  %92 : Float(240, 42, 42) = onnx::Softmax[axis=2](%91), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/ScaledDotProductAttentionLayer[attention_layer]\n",
      "  %93 : Float(240, 42, 42), %94 : Tensor = onnx::Dropout[ratio=0.1](%92), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/ScaledDotProductAttentionLayer[attention_layer]/Dropout[dropout]\n",
      "  %95 : Float(240, 42, 100) = onnx::MatMul(%93, %87), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/ScaledDotProductAttentionLayer[attention_layer]\n",
      "  %96 : Long() = onnx::Constant[value={3}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %97 : Long() = onnx::Constant[value={100}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %98 : int[] = prim::ListConstruct(%96, %51, %54, %97), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %99 : Float(3, 80, 42, 100) = onnx::Reshape(%95, %98), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %100 : Float(80, 42, 3, 100) = onnx::Transpose[perm=[1, 2, 0, 3]](%99), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %101 : Long() = onnx::Constant[value={-1}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %102 : int[] = prim::ListConstruct(%51, %54, %101), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %103 : Float(80, 42, 300) = onnx::Reshape(%100, %102), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %104 : Float(300!, 300!) = onnx::Transpose[perm=[1, 0]](%8), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/Linear[w_0]\n",
      "  %105 : Float(80, 42, 300) = onnx::MatMul(%103, %104), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/Linear[w_0]\n",
      "  %106 : Float(80, 42, 300), %107 : Tensor = onnx::Dropout[ratio=0.1](%105), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/Dropout[dropout]\n",
      "  %108 : Float(80, 42, 300) = onnx::Add(%106, %41), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %109 : Float(80, 42, 1) = onnx::ReduceMean[axes=[-1], keepdims=1](%108), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/LayerNorm[layer_norm]\n",
      "  %110 : Long() = onnx::Constant[value={-1}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/LayerNorm[layer_norm]\n",
      "  %111 : Long() = onnx::Constant[value={1}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/LayerNorm[layer_norm]\n",
      "  %112 : Long() = onnx::Constant[value={1}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/LayerNorm[layer_norm]\n",
      "  %113 : Float(80, 42, 1) = aten::std(%108, %110, %111, %112), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/LayerNorm[layer_norm]\n",
      "  %114 : Float(80, 42, 300) = onnx::Sub(%108, %109), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/LayerNorm[layer_norm]\n",
      "  %115 : Float(80, 42, 300) = onnx::Mul(%6, %114), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/LayerNorm[layer_norm]\n",
      "  %116 : Tensor = onnx::Constant[value={1e-06}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/LayerNorm[layer_norm]\n",
      "  %117 : Float(80, 42, 1) = onnx::Add(%113, %116), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/LayerNorm[layer_norm]\n",
      "  %118 : Float(80, 42, 300) = onnx::Div(%115, %117), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/LayerNorm[layer_norm]\n",
      "  %119 : Float(80, 42, 300) = onnx::Add(%118, %7), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/LayerNorm[layer_norm]\n",
      "  %120 : Float(300!, 2048!) = onnx::Transpose[perm=[1, 0]](%11), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/Linear\n",
      "  %121 : Float(80, 42, 2048) = onnx::MatMul(%119, %120), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/Linear\n",
      "  %122 : Float(80, 42, 2048) = onnx::Add(%121, %12), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/Linear\n",
      "  %123 : Float(80, 42, 2048) = onnx::Relu(%122), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock\n",
      "  %124 : Float(2048!, 300!) = onnx::Transpose[perm=[1, 0]](%13), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/Linear\n",
      "  %125 : Float(80, 42, 300) = onnx::MatMul(%123, %124), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/Linear\n",
      "  %126 : Float(80, 42, 300) = onnx::Add(%125, %14), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/Linear\n",
      "  %127 : Float(80, 42, 300), %128 : Tensor = onnx::Dropout[ratio=0.1](%126), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/Dropout\n",
      "  %129 : Float(80, 42, 300) = onnx::Add(%127, %119), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock\n",
      "  %130 : Float(80, 42, 1) = onnx::ReduceMean[axes=[-1], keepdims=1](%129), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/LayerNorm\n",
      "  %131 : Long() = onnx::Constant[value={-1}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/LayerNorm\n",
      "  %132 : Long() = onnx::Constant[value={1}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/LayerNorm\n",
      "  %133 : Long() = onnx::Constant[value={1}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/LayerNorm\n",
      "  %134 : Float(80, 42, 1) = aten::std(%129, %131, %132, %133), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/LayerNorm\n",
      "  %135 : Float(80, 42, 300) = onnx::Sub(%129, %130), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/LayerNorm\n",
      "  %136 : Float(80, 42, 300) = onnx::Mul(%9, %135), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/LayerNorm\n",
      "  %137 : Tensor = onnx::Constant[value={1e-06}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/LayerNorm\n",
      "  %138 : Float(80, 42, 1) = onnx::Add(%134, %137), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/LayerNorm\n",
      "  %139 : Float(80, 42, 300) = onnx::Div(%136, %138), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/LayerNorm\n",
      "  %140 : Float(80, 42, 300) = onnx::Add(%139, %10), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/LayerNorm\n",
      "  %141 : Float(300!, 300!) = onnx::Transpose[perm=[1, 0]](%17), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/Linear[query_projections]\n",
      "  %142 : Float(80, 42, 300) = onnx::MatMul(%140, %141), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/Linear[query_projections]\n",
      "  %143 : Float(300!, 300!) = onnx::Transpose[perm=[1, 0]](%18), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/Linear[key_projections]\n",
      "  %144 : Float(80, 42, 300) = onnx::MatMul(%140, %143), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/Linear[key_projections]\n",
      "  %145 : Float(300!, 300!) = onnx::Transpose[perm=[1, 0]](%19), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/Linear[value_projections]\n",
      "  %146 : Float(80, 42, 300) = onnx::MatMul(%140, %145), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/Linear[value_projections]\n",
      "  %147 : Long() = onnx::Constant[value={0}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %148 : Tensor = onnx::Shape(%140), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %149 : Long() = onnx::Gather[axis=0](%148, %147), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %150 : Long() = onnx::Constant[value={1}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %151 : Tensor = onnx::Shape(%140), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %152 : Long() = onnx::Gather[axis=0](%151, %150), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %153 : Long() = onnx::Constant[value={1}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %154 : Tensor = onnx::Shape(%140), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %155 : Long() = onnx::Gather[axis=0](%154, %153), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %156 : Long() = onnx::Constant[value={1}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %157 : Tensor = onnx::Shape(%140), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %158 : Long() = onnx::Gather[axis=0](%157, %156), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %159 : Long() = onnx::Constant[value={3}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %160 : Long() = onnx::Constant[value={100}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %161 : int[] = prim::ListConstruct(%149, %152, %159, %160), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %162 : Float(80, 42, 3, 100) = onnx::Reshape(%142, %161), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %163 : Long() = onnx::Constant[value={3}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %164 : Long() = onnx::Constant[value={100}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %165 : int[] = prim::ListConstruct(%149, %155, %163, %164), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %166 : Float(80, 42, 3, 100) = onnx::Reshape(%144, %165), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %167 : Long() = onnx::Constant[value={3}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %168 : Long() = onnx::Constant[value={100}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %169 : int[] = prim::ListConstruct(%149, %158, %167, %168), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %170 : Float(80, 42, 3, 100) = onnx::Reshape(%146, %169), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %171 : Float(3, 80, 42, 100) = onnx::Transpose[perm=[2, 0, 1, 3]](%162), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %172 : Long() = onnx::Constant[value={-1}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %173 : Long() = onnx::Constant[value={100}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %174 : int[] = prim::ListConstruct(%172, %152, %173), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %175 : Float(240, 42, 100) = onnx::Reshape(%171, %174), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %176 : Float(3, 80, 42, 100) = onnx::Transpose[perm=[2, 0, 1, 3]](%166), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %177 : Long() = onnx::Constant[value={-1}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %178 : Long() = onnx::Constant[value={100}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %179 : int[] = prim::ListConstruct(%177, %155, %178), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %180 : Float(240, 42, 100) = onnx::Reshape(%176, %179), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %181 : Float(3, 80, 42, 100) = onnx::Transpose[perm=[2, 0, 1, 3]](%170), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %182 : Long() = onnx::Constant[value={-1}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %183 : Long() = onnx::Constant[value={100}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %184 : int[] = prim::ListConstruct(%182, %158, %183), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %185 : Float(240, 42, 100) = onnx::Reshape(%181, %184), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %186 : Float(240!, 100!, 42!) = onnx::Transpose[perm=[0, 2, 1]](%180), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/ScaledDotProductAttentionLayer[attention_layer]\n",
      "  %187 : Float(240, 42, 42) = onnx::MatMul(%175, %186), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/ScaledDotProductAttentionLayer[attention_layer]\n",
      "  %188 : Tensor = onnx::Constant[value={8}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/ScaledDotProductAttentionLayer[attention_layer]\n",
      "  %189 : Float(240, 42, 42) = onnx::Div(%187, %188), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/ScaledDotProductAttentionLayer[attention_layer]\n",
      "  %190 : Float(240, 42, 42) = onnx::Softmax[axis=2](%189), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/ScaledDotProductAttentionLayer[attention_layer]\n",
      "  %191 : Float(240, 42, 42), %192 : Tensor = onnx::Dropout[ratio=0.1](%190), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/ScaledDotProductAttentionLayer[attention_layer]/Dropout[dropout]\n",
      "  %193 : Float(240, 42, 100) = onnx::MatMul(%191, %185), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/ScaledDotProductAttentionLayer[attention_layer]\n",
      "  %194 : Long() = onnx::Constant[value={3}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %195 : Long() = onnx::Constant[value={100}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %196 : int[] = prim::ListConstruct(%194, %149, %152, %195), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %197 : Float(3, 80, 42, 100) = onnx::Reshape(%193, %196), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %198 : Float(80, 42, 3, 100) = onnx::Transpose[perm=[1, 2, 0, 3]](%197), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %199 : Long() = onnx::Constant[value={-1}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %200 : int[] = prim::ListConstruct(%149, %152, %199), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %201 : Float(80, 42, 300) = onnx::Reshape(%198, %200), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %202 : Float(300!, 300!) = onnx::Transpose[perm=[1, 0]](%22), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/Linear[w_0]\n",
      "  %203 : Float(80, 42, 300) = onnx::MatMul(%201, %202), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/Linear[w_0]\n",
      "  %204 : Float(80, 42, 300), %205 : Tensor = onnx::Dropout[ratio=0.1](%203), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/Dropout[dropout]\n",
      "  %206 : Float(80, 42, 300) = onnx::Add(%204, %140), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]\n",
      "  %207 : Float(80, 42, 1) = onnx::ReduceMean[axes=[-1], keepdims=1](%206), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/LayerNorm[layer_norm]\n",
      "  %208 : Long() = onnx::Constant[value={-1}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/LayerNorm[layer_norm]\n",
      "  %209 : Long() = onnx::Constant[value={1}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/LayerNorm[layer_norm]\n",
      "  %210 : Long() = onnx::Constant[value={1}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/LayerNorm[layer_norm]\n",
      "  %211 : Float(80, 42, 1) = aten::std(%206, %208, %209, %210), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/LayerNorm[layer_norm]\n",
      "  %212 : Float(80, 42, 300) = onnx::Sub(%206, %207), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/LayerNorm[layer_norm]\n",
      "  %213 : Float(80, 42, 300) = onnx::Mul(%20, %212), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/LayerNorm[layer_norm]\n",
      "  %214 : Tensor = onnx::Constant[value={1e-06}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/LayerNorm[layer_norm]\n",
      "  %215 : Float(80, 42, 1) = onnx::Add(%211, %214), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/LayerNorm[layer_norm]\n",
      "  %216 : Float(80, 42, 300) = onnx::Div(%213, %215), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/LayerNorm[layer_norm]\n",
      "  %217 : Float(80, 42, 300) = onnx::Add(%216, %21), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/MultiHeadedSelfAttentionLayer[self_attention_layer]/LayerNorm[layer_norm]\n",
      "  %218 : Float(300!, 2048!) = onnx::Transpose[perm=[1, 0]](%25), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/Linear\n",
      "  %219 : Float(80, 42, 2048) = onnx::MatMul(%217, %218), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/Linear\n",
      "  %220 : Float(80, 42, 2048) = onnx::Add(%219, %26), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/Linear\n",
      "  %221 : Float(80, 42, 2048) = onnx::Relu(%220), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock\n",
      "  %222 : Float(2048!, 300!) = onnx::Transpose[perm=[1, 0]](%27), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/Linear\n",
      "  %223 : Float(80, 42, 300) = onnx::MatMul(%221, %222), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/Linear\n",
      "  %224 : Float(80, 42, 300) = onnx::Add(%223, %28), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/Linear\n",
      "  %225 : Float(80, 42, 300), %226 : Tensor = onnx::Dropout[ratio=0.1](%224), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/Dropout\n",
      "  %227 : Float(80, 42, 300) = onnx::Add(%225, %217), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock\n",
      "  %228 : Float(80, 42, 1) = onnx::ReduceMean[axes=[-1], keepdims=1](%227), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/LayerNorm\n",
      "  %229 : Long() = onnx::Constant[value={-1}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/LayerNorm\n",
      "  %230 : Long() = onnx::Constant[value={1}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/LayerNorm\n",
      "  %231 : Long() = onnx::Constant[value={1}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/LayerNorm\n",
      "  %232 : Float(80, 42, 1) = aten::std(%227, %229, %230, %231), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/LayerNorm\n",
      "  %233 : Float(80, 42, 300) = onnx::Sub(%227, %228), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/LayerNorm\n",
      "  %234 : Float(80, 42, 300) = onnx::Mul(%23, %233), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/LayerNorm\n",
      "  %235 : Tensor = onnx::Constant[value={1e-06}](), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/LayerNorm\n",
      "  %236 : Float(80, 42, 1) = onnx::Add(%232, %235), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/LayerNorm\n",
      "  %237 : Float(80, 42, 300) = onnx::Div(%234, %236), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/LayerNorm\n",
      "  %238 : Float(80, 42, 300) = onnx::Add(%237, %24), scope: TransformerTagger/TransformerEncoder[encoder]/EncoderBlock/LayerNorm\n",
      "  %239 : Float(300!, 10!) = onnx::Transpose[perm=[1, 0]](%33), scope: TransformerTagger/SoftmaxOutputLayer[taggingLayer]/Linear[output_projection]\n",
      "  %240 : Float(80, 42, 10) = onnx::MatMul(%238, %239), scope: TransformerTagger/SoftmaxOutputLayer[taggingLayer]/Linear[output_projection]\n",
      "  %241 : Float(80, 42, 10) = onnx::Add(%240, %34), scope: TransformerTagger/SoftmaxOutputLayer[taggingLayer]/Linear[output_projection]\n",
      "  %242 : Float(80, 42, 10) = onnx::LogSoftmax[axis=2](%241), scope: TransformerTagger/SoftmaxOutputLayer[taggingLayer]\n",
      "  return (%242);\n",
      "}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "optimizer = get_default_optimizer(model, hyper_parameters)\n",
    "trainer = Trainer(target_size, \n",
    "                    model,\n",
    "                    loss,\n",
    "                    optimizer,\n",
    "                    hyper_parameters,\n",
    "                    conll2003['iters'],\n",
    "                    experiment_name,\n",
    "                    log_every_xth_iteration=-1,\n",
    "                    enable_tensorboard=True,\n",
    "                    dummy_input=conll2003['dummy_input'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pre_training - DEBUG - train with cuda support\n",
      "pre_training - INFO - 176 Iterations per epoch with batch size of 80\n",
      "pre_training - INFO - START training.\n",
      "\n",
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ff732f20a2d343ca9e7f9f0644975d45",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d95edbc175644584bd0d0c759861df6f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time\n",
      "1\t176\t0.843\t\t0.722\t\t0.771\t\t0.834\t\t2.13m - 2.1m / 0.0m\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "700a7236a59943be9ec5f9f94bc283d7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4bb2487f076040ccbb4dc98968f63450",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\t352\t0.114\t\t0.383\t\t0.880\t\t0.909\t\t2.15m - 4.3m / 10.7m\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "98ead353fe3b429eba26f02d139b1c73",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "faa10ac3b48748099e8f169f05961700",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "464db32aee434fd6be6f48af1b64f759",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-11-fa31de641ef0>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtrainer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m5\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\models\\transformer\\train.py\u001b[0m in \u001b[0;36mtrain\u001b[1;34m(self, num_epochs, use_cuda, perform_evaluation)\u001b[0m\n\u001b[0;32m    466\u001b[0m                 \u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbatch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0minputs_word\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlabels\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    467\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 468\u001b[1;33m                 \u001b[0mtrain_loss\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_step\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    469\u001b[0m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_log_scalar\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrain_loss_history\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtrain_loss\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'loss'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'train'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0miteration\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    470\u001b[0m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_log_scalar\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'lr'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m''\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0miteration\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\models\\transformer\\train.py\u001b[0m in \u001b[0;36m_step\u001b[1;34m(self, input, target)\u001b[0m\n\u001b[0;32m    235\u001b[0m         \u001b[1;31m# preform training step\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    236\u001b[0m         \u001b[0mloss\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 237\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    238\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    239\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mloss\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\optimizer.py\u001b[0m in \u001b[0;36mstep\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m     23\u001b[0m             \u001b[0mp\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'lr'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mrate\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     24\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_rate\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mrate\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 25\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     26\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     27\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mzero_grad\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\torch\\optim\\adam.py\u001b[0m in \u001b[0;36mstep\u001b[1;34m(self, closure)\u001b[0m\n\u001b[0;32m     92\u001b[0m                 \u001b[1;31m# Decay the first and second moment running average coefficient\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     93\u001b[0m                 \u001b[0mexp_avg\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmul_\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbeta1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0madd_\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m1\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0mbeta1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgrad\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 94\u001b[1;33m                 \u001b[0mexp_avg_sq\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmul_\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbeta2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0maddcmul_\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m1\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0mbeta2\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgrad\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgrad\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     95\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[0mamsgrad\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     96\u001b[0m                     \u001b[1;31m# Maintains the maximum of all 2nd moment running avg. till now\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "result = trainer.train(5, True, False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plt.rcParams['figure.figsize'] = [10, 10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = result['model']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result_labels = trainer.classify_sentence('I was born in 1993 in Stuttgart')\n",
    "\n",
    "\n",
    "\n",
    "print(result_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tr = trainer.text_reverser[1]\n",
    "lr = trainer.label_reverser\n",
    "\n",
    "test_sentence = ['china', 'controlled', 'most', 'of', 'the', 'match']\n",
    "test_sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_sentence = tr.preprocess('My name is Felix and I was born in Germany')\n",
    "\n",
    "#test_sentence = tr.preprocess('china controlled most of the match on 1993')\n",
    "test_sentence = [x.strip(' ') for x in test_sentence]\n",
    "test_sentence = [test_sentence]\n",
    "test_sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "#test = tr.preprocess('china controlled most of the match')\n",
    "#print(test)\n",
    "\n",
    "#test_sentence = [['china', 'controlled', 'most', 'of', 'the', 'match']]\n",
    "x = tr.process(test_sentence)\n",
    "\n",
    "print(\"X TENSOR \",x)\n",
    "print('X Size', x.size())\n",
    "print(\"Reversed X\", tr.reverse(x))\n",
    "x = x.cuda()\n",
    "y_hat = model.predict(x)\n",
    "y_hat_label = lr.reverse(y_hat)\n",
    "print(y_hat_label)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainer.tb_writer = None\n",
    "trainer.enable_tensorboard = False\n",
    "evaluation_results = trainer.perform_final_evaluation()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "c_matrix = evaluation_results[1][2]\n",
    "c_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = plot_confusion_matrix(c_matrix, class_labels)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = plot_confusion_matrix(c_matrix, class_labels, normalize=True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# predict now to see model in final state\n",
    "from IPython.display import display, HTML\n",
    "\n",
    "df = predict_some_examples_to_df(model, conll2003['iters'][2], num_samples=800)\n",
    "display(HTML(df.to_html()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = predict_some_examples_to_df(model, conll2003['iters'][1], num_samples=800)\n",
    "display(HTML(df.to_html()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "df = predict_some_examples_to_df(model, conll2003['iters'][0], num_samples=800)\n",
    "display(HTML(df.to_html()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "a = np.array([\n",
    "    np.array([[1, 1], [1, 1]]),\n",
    "    np.array([[2, 2], [-2, -3]])\n",
    "])\n",
    "print(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "a.sum(axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#df = predict_some_examples_to_df(model, test_sample_iter)\n",
    "#print(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "(tr_loss, tr_f1) = result['result_train']\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}