e2e_el.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "e2e_el.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [
        "mpfjaJPy-fel",
        "E0hd7jfM-NTh",
        "DNxRHDO4wz0J"
      ]
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RFDWlKWYvu8M",
        "colab_type": "text"
      },
      "source": [
        "# Entity Linking Baseline on AIDA-YAGO Dataset"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jKbDUdkw9sAT",
        "colab_type": "text"
      },
      "source": [
        "# 1. Load Libs & Data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mpfjaJPy-fel",
        "colab_type": "text"
      },
      "source": [
        "## Install Needed Libs"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f4rJfvYPzy4h",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "! pip install gensim"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "O7Gegg82-oFC",
        "colab_type": "text"
      },
      "source": [
        "## Mount Drive"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1uoOxX6b-zhL",
        "colab_type": "code",
        "outputId": "6f03ff7d-72c5-4054-dfb0-f499df45b037",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        }
      },
      "source": [
        "from google.colab import drive\n",
        "\n",
        "drive.mount('/content/drive')\n",
        "drive_path = 'drive/My Drive/e2e_el/'"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=email%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdocs.test%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive.photos.readonly%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fpeopleapi.readonly&response_type=code\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "E0hd7jfM-NTh",
        "colab_type": "text"
      },
      "source": [
        "## Preprocess Wikipedia Dump"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RBG2L2aVyN8G",
        "colab_type": "code",
        "outputId": "75093e1d-ccb1-4fdd-9699-a33dae7138fc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 224
        }
      },
      "source": [
        "! wget https://dumps.wikimedia.org/archive/enwiki/20100312/enwiki-20100312-pages-articles.xml.bz2\n",
        "! cp enwiki-20100312-pages-articles.xml.bz2 drive/My\\ Drive/e2e_el/enwiki.xml.bz2"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2019-07-15 16:13:12--  https://dumps.wikimedia.org/archive/enwiki/20100312/enwiki-20100312-pages-articles.xml.bz2\n",
            "Resolving dumps.wikimedia.org (dumps.wikimedia.org)... 208.80.155.106, 2620:0:861:4:208:80:155:106\n",
            "Connecting to dumps.wikimedia.org (dumps.wikimedia.org)|208.80.155.106|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 6117881141 (5.7G) [application/octet-stream]\n",
            "Saving to: ‘enwiki-20100312-pages-articles.xml.bz2’\n",
            "\n",
            "enwiki-20100312-pag 100%[===================>]   5.70G  1.97MB/s    in 49m 29s \n",
            "\n",
            "2019-07-15 17:02:42 (1.97 MB/s) - ‘enwiki-20100312-pages-articles.xml.bz2’ saved [6117881141/6117881141]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OeqkVDVO_B0F",
        "colab_type": "code",
        "outputId": "65e12d95-f1fc-4bfb-d92c-44e19fd82efc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "! cp drive/My\\ Drive/e2e_el/enwiki.xml.bz2 enwiki.xml.bz2\n",
        "! bzip2 -dk -v enwiki.xml.bz2"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  enwiki.xml.bz2: done\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZHXGyo7Q9KD4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "! python WikiExtractor.py enwiki.xml -o enwiki -l --no_templates --processes 4 -q"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MgU7ahEt9dj1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# python 2.7\n",
        "\n",
        "import os\n",
        "\n",
        "def merge(in_dir, out_path):\n",
        "        \n",
        "    with open(out_path, 'w') as out_file:\n",
        "        \n",
        "        for sub_dir in sorted(os.listdir(in_dir)):\n",
        "            print sub_dir, \n",
        "            file_dir = in_dir + '/' + sub_dir\n",
        "            \n",
        "            for in_name in sorted(os.listdir(file_dir)):\n",
        "                with open(file_dir + '/' + in_name, 'r') as in_file:\n",
        "                    for line in in_file:\n",
        "                        out_file.write(line)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D4bScOIC9egE",
        "colab_type": "code",
        "outputId": "fae6b359-08c9-4de3-c2fd-bcdb6a5a16ef",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "merge(in_dir='enwiki', out_path='enwiki_full.txt')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "AA AB AC AD AE AF AG AH AI AJ AK AL AM AN AO AP AQ AR AS AT AU AV AW AX AY AZ BA BB BC BD BE BF BG BH BI BJ BK BL BM BN BO BP BQ BR BS BT BU BV BW BX BY BZ CA CB CC CD CE CF CG CH CI CJ CK CL CM CN CO CP CQ CR CS CT CU CV CW CX CY CZ DA DB DC DD DE DF DG DH DI DJ\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vjuwD689VYbx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "! cp enwiki_full.txt drive/My\\ Drive/e2e_el/enwiki_links_full.txt"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iecmhAg3s0Wb",
        "colab_type": "text"
      },
      "source": [
        "## Load Pretrained FastText"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j78ZsqPOoSA0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# ! wget https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.en.zip\n",
        "# ! wget https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.simple.zip\n",
        "# ! unzip wiki.simple.zip"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ofn60TjOtk7S",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# ! cp wiki.simple.bin drive/My\\ Drive/e2e_el/wiki.simple.bin\n",
        "! cp drive/My\\ Drive/e2e_el/wiki.simple.bin wiki.simple.bin"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8xm_Feu45myI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from gensim.models.wrappers import FastText\n",
        "\n",
        "ft = FastText.load_fasttext_format('wiki.simple.bin')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ohQePvLOm7cq",
        "colab_type": "text"
      },
      "source": [
        "## Load Wiki Entities"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nbj-ZaZ1qEDq",
        "colab_type": "code",
        "outputId": "f6e6fa3f-ef5a-4750-fa87-0c850508550a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "! cp drive/My\\ Drive/e2e_el/enwiki_links_full.txt enwiki_full.txt\n",
        "# ! cp drive/My\\ Drive/e2e_el/entities.pickle entities.pickle "
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "cp: cannot stat 'drive/My Drive/e2e_el/entities.pickle': No such file or directory\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aMdW2cCOwNIH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import sys\n",
        "from importlib import reload\n",
        "\n",
        "sys.path.append('/content/drive/My Drive/colab_utils/')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MLOPwwm1ECt4",
        "colab_type": "code",
        "outputId": "fed2d875-18ae-4714-ec32-fba8026f85cc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "import process_wiki\n",
        "\n",
        "process_wiki = reload(process_wiki)\n",
        "\n",
        "ent_max = 1000000     # 10 ** 7 or 5500000\n",
        "\n",
        "name2wikiid, wikiid2name, id2wikiid, wikiid2id = process_wiki.load_entity_id_maps(\n",
        "    ent_pickle_path='entities.pickle',\n",
        "    ent_path='enwiki_full.txt',\n",
        "    ent_max=ent_max)\n",
        "\n",
        "ent_size = len(wikiid2id)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "loaded entities from entities.pickle\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Aa6hnd_6-hlA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# ! cp entities.pickle drive/My\\ Drive/e2e_el/entities.pickle \n",
        "# ! cp drive/My\\ Drive/e2e_el/mentions.pickle mentions.pickle"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MVV5ouymw3Hb",
        "colab_type": "code",
        "outputId": "c4f1c125-edc4-4be0-bd46-3caa1114c1bb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        }
      },
      "source": [
        "%%time\n",
        "\n",
        "import gen_mention\n",
        "\n",
        "gen_mention = reload(gen_mention)\n",
        "\n",
        "wikiid2mentions, ignored = gen_mention.load_entity_mentions_map(\n",
        "    ent_path='enwiki_full.txt', \n",
        "    name2wikiid=name2wikiid, \n",
        "    men_pickle_path='mentions.pickle', \n",
        "    ent_max=ent_max)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "loaded mentions from mentions.pickle\n",
            "CPU times: user 6.67 s, sys: 1.02 s, total: 7.69 s\n",
            "Wall time: 7.75 s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cE30uGUuWr_9",
        "colab_type": "code",
        "outputId": "71066336-09be-407d-9dab-1c4dcb745a8f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# ! cp mentions.pickle drive/My\\ Drive/e2e_el/mentions.pickle\n",
        "len(ignored), len(wikiid2mentions), len(wikiid2id)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(0, 667510, 1000000)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fw1Z1uqn9Ach",
        "colab_type": "code",
        "outputId": "0e7c1e66-4f64-4b6d-b877-6e3bc8274c2e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        }
      },
      "source": [
        "%%time\n",
        "\n",
        "import numpy as np\n",
        "\n",
        "gen_mention = reload(gen_mention)\n",
        "\n",
        "emb_size = 300\n",
        "\n",
        "ent_men_emb = gen_mention.gen_entity_mention_vec(\n",
        "    wikiid2mentions,\n",
        "    id2wikiid, wikiid2name,\n",
        "    emb_model=ft, emb_size=emb_size,\n",
        "    alpha=0.2)\n",
        "\n",
        "ent_men_emb.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[===================================================] 100% done!\n",
            "CPU times: user 11min 7s, sys: 12.2 s, total: 11min 20s\n",
            "Wall time: 11min 19s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "43WSIOV3hKYY",
        "colab_type": "code",
        "outputId": "1d9a229a-3982-4311-93f3-75f7de7a5a02",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "wikiid2name[id2wikiid[0]], ft.similar_by_vector(ent_men_emb[0])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "('UNK_E',\n",
              " [('hcalendar', 0.0),\n",
              "  ('chía', 0.0),\n",
              "  ('deportiva', 0.0),\n",
              "  ('tufnell', 0.0),\n",
              "  ('frasin', 0.0),\n",
              "  ('jumpsuit', 0.0),\n",
              "  ('berankis', 0.0),\n",
              "  ('evolutionist', 0.0),\n",
              "  ('mdv', 0.0),\n",
              "  ('morgen', 0.0)])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 42
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tgupC8y2-UNe",
        "colab_type": "text"
      },
      "source": [
        "## Prepare AIDA-YAGO Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GlzcDwp_sTdU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "! cp drive/My\\ Drive/e2e_el/aida-yago2-dataset.tsv aida-yago2-dataset.tsv\n",
        "# ! wget http://resources.mpi-inf.mpg.de/yago-naga/aida/download/aida_means.tsv.bz2\n",
        "# ! bzip2 -dk aida_means.tsv.bz2"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e3eSweffwPRX",
        "colab_type": "code",
        "outputId": "5b1cfacd-0534-4c4e-a00e-d844dc7875b2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "import sys\n",
        "import process_aida\n",
        "from importlib import reload\n",
        "\n",
        "sys.path.insert(0, '/content/drive/My Drive/colab_utils/')\n",
        "process_aida = reload(process_aida)\n",
        "\n",
        "process_aida.split_aida(aida_path='aida-yago2-dataset.tsv')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "aida splitted to aida-train.tsv, aida-testa.tsv, aida-testb.tsv\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qLfuQhwHbhqj",
        "colab_type": "code",
        "outputId": "66148da0-0dc2-4a2f-fa27-2e2562fed93e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "import process_aida\n",
        "process_aida = reload(process_aida)\n",
        "\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from keras.utils import to_categorical\n",
        "\n",
        "\n",
        "def pad_data(x, iob, y, w, l, include_mention=True, ib_w=2):\n",
        "  \n",
        "  data_x = pad_sequences(x, maxlen=max(l), value=0, dtype=np.float16,\n",
        "                             padding='post', truncating='post')\n",
        "  data_iob = pad_sequences(iob, maxlen=max(l), value=0, dtype=np.int32,\n",
        "                         padding='post', truncating='post')\n",
        "  \n",
        "  if include_mention:\n",
        "    \n",
        "    data_w = {\n",
        "    'iob_output': np.ones(data_iob.shape),\n",
        "    'mention_output': np.ones(data_iob.shape),\n",
        "    }\n",
        "    data_w['iob_output'][data_iob > 0] = ib_w\n",
        "    data_w['mention_output'][data_iob > 0] = ib_w\n",
        "\n",
        "    data_iob = to_categorical(data_iob, num_classes=3, dtype=np.int32)\n",
        "    data_y = pad_sequences(y, maxlen=max(l), value=0, dtype=np.float16,\n",
        "                           padding='post', truncating='post')\n",
        "  \n",
        "    return data_x, [data_iob, data_y], data_w\n",
        "  \n",
        "  else:\n",
        "    \n",
        "    data_w = np.ones(data_iob.shape)\n",
        "    data_w[data_iob > 0] = ib_w\n",
        "\n",
        "    data_iob = to_categorical(data_iob, num_classes=3, dtype=np.int32)\n",
        "\n",
        "    return data_x, data_iob, data_w\n",
        "\n",
        "  \n",
        "def padded_gen(data_path, include_mention=True, batch_size=32, begin_inside_weight=2):\n",
        "  \n",
        "  global ft, wikiid2id, ent_men_emb\n",
        "  \n",
        "  while True:\n",
        "    gen = process_aida.gen_el_data_vecs(aida_path=data_path,\n",
        "                                            word_vecs=ft, wiki2id=wikiid2id,\n",
        "                                            ent_vecs=ent_men_emb)\n",
        "\n",
        "    data_x, data_iob, data_y, data_w, lengths = [], [], [], [], []\n",
        "\n",
        "    size = 0\n",
        "\n",
        "    for (tokens, iobs, mentions) in gen:\n",
        "      lengths.append(len(tokens))\n",
        "      data_x.append(tokens)\n",
        "      data_iob.append(iobs)\n",
        "      data_y.append(mentions)\n",
        "\n",
        "      size += 1\n",
        "\n",
        "      if size % batch_size == 0:\n",
        "\n",
        "        data_x, data_y, data_w = pad_data(\n",
        "            data_x, data_iob, data_y, data_w, lengths, include_mention, begin_inside_weight)\n",
        "\n",
        "        yield data_x, data_y, data_w\n",
        "\n",
        "        data_x, data_iob, data_y, data_w, lengths = [], [], [], [], []\n",
        "\n",
        "    if size % batch_size != 0:\n",
        "\n",
        "      data_x, data_y, data_w = pad_data(\n",
        "        data_x, data_iob, data_y, data_w, lengths)\n",
        "\n",
        "      yield data_x, data_y, data_w"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vbWxq6lPQ_FT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "for x, y, w in padded_gen('aida-testa.tsv', batch_size=32):\n",
        "  print(y[1])\n",
        "  break"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6vTT6M2799Ed",
        "colab_type": "text"
      },
      "source": [
        "# 2. Define Baseline Model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DNxRHDO4wz0J",
        "colab_type": "text"
      },
      "source": [
        "## extras"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i0VF6IMvJRtV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from keras import backend as K\n",
        "\n",
        "def f1(y_true, y_pred):\n",
        "  \n",
        "    def recall(y_true, y_pred):\n",
        "        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
        "        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n",
        "        recall = true_positives / (possible_positives + K.epsilon())\n",
        "        return recall\n",
        "\n",
        "    def precision(y_true, y_pred):\n",
        "        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
        "        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n",
        "        precision = true_positives / (predicted_positives + K.epsilon())\n",
        "        return precision\n",
        "      \n",
        "    precision = precision(y_true, y_pred)\n",
        "    recall = recall(y_true, y_pred)\n",
        "    return 2*((precision*recall)/(precision+recall+K.epsilon()))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ddYT64ntRT4t",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import keras.backend as K\n",
        "\n",
        "iobs = K.random_normal_variable(shape=(20, 3), mean=0, scale=1)\n",
        "words = K.random_normal_variable(shape=(20, 100), mean=0, scale=1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BwFxsUabRb8v",
        "colab_type": "code",
        "outputId": "bb669cd1-464d-4e3d-a200-5122d7f88e6b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "xy = K.dot(x, y)\n",
        "\n",
        "with K.get_session().as_default():\n",
        "#   print(iobs.eval())\n",
        "  ibs = iobs[:, 1:]\n",
        "  max_ib = K.max(iobs, axis=-1)\n",
        "  expanded_max_ib = K.repeat_elements(K.expand_dims(max_ib, axis=-1), rep=K.int_shape(words)[-1], axis=-1)\n",
        "  print(max_ib.eval().shape, expanded_max_ib.eval().shape)\n",
        "  output = expanded_max_ib * words\n",
        "  print(output.eval().shape)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(20,) (20, 100)\n",
            "(20, 100)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AyeAC4AOrgUY",
        "colab_type": "text"
      },
      "source": [
        "## Design & Train Mention Detector"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EGO24-ZFOjcs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import keras.backend as K\n",
        "\n",
        "def filter_words_by_iob(x):\n",
        "  \n",
        "  words = x[0]\n",
        "  ibs = x[1][:, :, 1:]  # batch_idx, word_idx, iob\n",
        "  \n",
        "  emb_size = K.int_shape(words)[-1]\n",
        "  \n",
        "  max_ib = K.max(ibs, axis=-1)\n",
        "  expanded_max_ib = K.expand_dims(max_ib, axis=-1)\n",
        "  repeated_max_ib = K.repeat_elements(expanded_max_ib, rep=emb_size, axis=-1)\n",
        "  \n",
        "  return expanded_max_ib * words"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tVO4Edzgy-X0",
        "colab_type": "code",
        "outputId": "e6e38707-80cd-48f6-d09d-3360cb09cb46",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 649
        }
      },
      "source": [
        "from keras.models import Model, Input\n",
        "from keras.layers import LSTM, GRU, Embedding, Dense, TimeDistributed, Dropout, Bidirectional, Concatenate, Lambda\n",
        "\n",
        "\n",
        "word_embs = Input(shape=(None, 300), name='input')\n",
        "x = Bidirectional(GRU(units=300, return_sequences=True, recurrent_dropout=0.1), name='bigru')(word_embs)\n",
        "x = Bidirectional(LSTM(units=30, return_sequences=True, recurrent_dropout=0.1), name='bilstm')(x)\n",
        "iob_output = TimeDistributed(Dense(3, activation='softmax'), name='iob_output')(x)\n",
        "\n",
        "filtered_words = Lambda(filter_words_by_iob, name='filtered_words')([word_embs, iob_output])\n",
        "mention_output = LSTM(units=300, return_sequences=True, recurrent_dropout=0.1, name='mention_output')(filtered_words)\n",
        "\n",
        "losses = {\n",
        "\t'iob_output': 'categorical_crossentropy',\n",
        "\t'mention_output': 'mse',\n",
        "}\n",
        "lossWeights = {'iob_output': 1.0, 'mention_output': 10.0}\n",
        "\n",
        "\n",
        "model = Model(word_embs, [iob_output, mention_output])\n",
        "\n",
        "model.compile(optimizer='adam', loss=losses, loss_weights=lossWeights,\n",
        "              sample_weight_mode='temporal')\n",
        "model.summary()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING: Logging before flag parsing goes to stderr.\n",
            "W0718 19:31:56.470701 139814515033984 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:517: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
            "\n",
            "W0718 19:31:56.512042 139814515033984 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:74: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n",
            "\n",
            "W0718 19:31:56.514128 139814515033984 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:4138: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n",
            "\n",
            "W0718 19:31:56.862503 139814515033984 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:133: The name tf.placeholder_with_default is deprecated. Please use tf.compat.v1.placeholder_with_default instead.\n",
            "\n",
            "W0718 19:31:56.872943 139814515033984 deprecation.py:506] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
            "W0718 19:31:59.043739 139814515033984 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/optimizers.py:790: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
            "\n",
            "W0718 19:31:59.069330 139814515033984 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3295: The name tf.log is deprecated. Please use tf.math.log instead.\n",
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input (InputLayer)              (None, None, 300)    0                                            \n",
            "__________________________________________________________________________________________________\n",
            "bigru (Bidirectional)           (None, None, 600)    1081800     input[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "bilstm (Bidirectional)          (None, None, 60)     151440      bigru[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "iob_output (TimeDistributed)    (None, None, 3)      183         bilstm[0][0]                     \n",
            "__________________________________________________________________________________________________\n",
            "filtered_words (Lambda)         (None, None, 300)    0           input[0][0]                      \n",
            "                                                                 iob_output[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "mention_output (LSTM)           (None, None, 300)    721200      filtered_words[0][0]             \n",
            "==================================================================================================\n",
            "Total params: 1,954,623\n",
            "Trainable params: 1,954,623\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hswP6hrHK4aX",
        "colab_type": "code",
        "outputId": "dd397857-daf0-4164-c7b3-e97187890db7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 768
        }
      },
      "source": [
        "train_path = 'aida-train.tsv'\n",
        "valid_path = 'aida-testa.tsv'\n",
        "\n",
        "history = model.fit_generator(\n",
        "    padded_gen(train_path, batch_size=32, begin_inside_weight=4),\n",
        "    steps_per_epoch = 30,\n",
        "    validation_data=padded_gen(valid_path, batch_size=32, begin_inside_weight=4),\n",
        "    validation_steps=7,\n",
        "    epochs=20, verbose=1,\n",
        "    initial_epoch=0)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "W0718 19:31:59.383495 139814515033984 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_grad.py:1250: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/20\n",
            "30/30 [==============================] - 164s 5s/step - loss: 0.6945 - iob_output_loss: 0.6334 - mention_output_loss: 0.0061 - val_loss: 0.3143 - val_iob_output_loss: 0.2686 - val_mention_output_loss: 0.0046\n",
            "Epoch 2/20\n",
            "30/30 [==============================] - 153s 5s/step - loss: 0.2912 - iob_output_loss: 0.2368 - mention_output_loss: 0.0054 - val_loss: 0.2004 - val_iob_output_loss: 0.1589 - val_mention_output_loss: 0.0041\n",
            "Epoch 3/20\n",
            "30/30 [==============================] - 154s 5s/step - loss: 0.2162 - iob_output_loss: 0.1655 - mention_output_loss: 0.0051 - val_loss: 0.1704 - val_iob_output_loss: 0.1307 - val_mention_output_loss: 0.0040\n",
            "Epoch 4/20\n",
            "30/30 [==============================] - 153s 5s/step - loss: 0.1887 - iob_output_loss: 0.1402 - mention_output_loss: 0.0049 - val_loss: 0.1532 - val_iob_output_loss: 0.1148 - val_mention_output_loss: 0.0038\n",
            "Epoch 5/20\n",
            "30/30 [==============================] - 153s 5s/step - loss: 0.1732 - iob_output_loss: 0.1263 - mention_output_loss: 0.0047 - val_loss: 0.1384 - val_iob_output_loss: 0.1010 - val_mention_output_loss: 0.0037\n",
            "Epoch 6/20\n",
            "30/30 [==============================] - 153s 5s/step - loss: 0.1571 - iob_output_loss: 0.1115 - mention_output_loss: 0.0046 - val_loss: 0.1268 - val_iob_output_loss: 0.0902 - val_mention_output_loss: 0.0037\n",
            "Epoch 7/20\n",
            "30/30 [==============================] - 153s 5s/step - loss: 0.1456 - iob_output_loss: 0.1011 - mention_output_loss: 0.0045 - val_loss: 0.1205 - val_iob_output_loss: 0.0846 - val_mention_output_loss: 0.0036\n",
            "Epoch 8/20\n",
            "30/30 [==============================] - 153s 5s/step - loss: 0.1359 - iob_output_loss: 0.0924 - mention_output_loss: 0.0044 - val_loss: 0.1163 - val_iob_output_loss: 0.0810 - val_mention_output_loss: 0.0035\n",
            "Epoch 9/20\n",
            "30/30 [==============================] - 153s 5s/step - loss: 0.1279 - iob_output_loss: 0.0853 - mention_output_loss: 0.0043 - val_loss: 0.1104 - val_iob_output_loss: 0.0754 - val_mention_output_loss: 0.0035\n",
            "Epoch 10/20\n",
            "30/30 [==============================] - 151s 5s/step - loss: 0.1212 - iob_output_loss: 0.0793 - mention_output_loss: 0.0042 - val_loss: 0.1088 - val_iob_output_loss: 0.0743 - val_mention_output_loss: 0.0034\n",
            "Epoch 11/20\n",
            "30/30 [==============================] - 151s 5s/step - loss: 0.1160 - iob_output_loss: 0.0748 - mention_output_loss: 0.0041 - val_loss: 0.1074 - val_iob_output_loss: 0.0732 - val_mention_output_loss: 0.0034\n",
            "Epoch 12/20\n",
            "30/30 [==============================] - 151s 5s/step - loss: 0.1109 - iob_output_loss: 0.0704 - mention_output_loss: 0.0041 - val_loss: 0.1060 - val_iob_output_loss: 0.0720 - val_mention_output_loss: 0.0034\n",
            "Epoch 13/20\n",
            "30/30 [==============================] - 152s 5s/step - loss: 0.1048 - iob_output_loss: 0.0649 - mention_output_loss: 0.0040 - val_loss: 0.1005 - val_iob_output_loss: 0.0668 - val_mention_output_loss: 0.0034\n",
            "Epoch 14/20\n",
            "30/30 [==============================] - 151s 5s/step - loss: 0.1034 - iob_output_loss: 0.0638 - mention_output_loss: 0.0040 - val_loss: 0.1010 - val_iob_output_loss: 0.0674 - val_mention_output_loss: 0.0034\n",
            "Epoch 15/20\n",
            "30/30 [==============================] - 151s 5s/step - loss: 0.1039 - iob_output_loss: 0.0649 - mention_output_loss: 0.0039 - val_loss: 0.1028 - val_iob_output_loss: 0.0694 - val_mention_output_loss: 0.0033\n",
            "Epoch 16/20\n",
            "30/30 [==============================] - 151s 5s/step - loss: 0.0995 - iob_output_loss: 0.0609 - mention_output_loss: 0.0039 - val_loss: 0.1120 - val_iob_output_loss: 0.0785 - val_mention_output_loss: 0.0033\n",
            "Epoch 17/20\n",
            "30/30 [==============================] - 151s 5s/step - loss: 0.0919 - iob_output_loss: 0.0538 - mention_output_loss: 0.0038 - val_loss: 0.1041 - val_iob_output_loss: 0.0708 - val_mention_output_loss: 0.0033\n",
            "Epoch 18/20\n",
            "30/30 [==============================] - 151s 5s/step - loss: 0.0875 - iob_output_loss: 0.0497 - mention_output_loss: 0.0038 - val_loss: 0.1078 - val_iob_output_loss: 0.0749 - val_mention_output_loss: 0.0033\n",
            "Epoch 19/20\n",
            "30/30 [==============================] - 151s 5s/step - loss: 0.0831 - iob_output_loss: 0.0455 - mention_output_loss: 0.0038 - val_loss: 0.1046 - val_iob_output_loss: 0.0718 - val_mention_output_loss: 0.0033\n",
            "Epoch 20/20\n",
            "30/30 [==============================] - 151s 5s/step - loss: 0.0802 - iob_output_loss: 0.0430 - mention_output_loss: 0.0037 - val_loss: 0.1092 - val_iob_output_loss: 0.0765 - val_mention_output_loss: 0.0033\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wDCqkz9R1oFI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model.save('el_mention_model.h5')\n",
        "! cp el_mention_model.h5 drive/My\\ Drive/e2e_el/el_mention_model.h5"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nIkW96z8fP5z",
        "colab_type": "code",
        "outputId": "e747468c-b3ea-4312-85f4-f70f1624e8f0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 298
        }
      },
      "source": [
        "from matplotlib import pyplot as plt\n",
        "from pylab import rcParams\n",
        "\n",
        "rcParams['figure.figsize'] = 12, 4\n",
        "\n",
        "plt.subplot(1, 2, 1)\n",
        "plt.title('Mention Loss per epoch')\n",
        "plt.plot(history.history['mention_output_loss'], 'b.')\n",
        "plt.plot(history.history['val_mention_output_loss'], 'b-')\n",
        "\n",
        "plt.subplot(1, 2, 2)\n",
        "plt.title('IOB Loss per epoch')\n",
        "plt.plot(history.history['iob_output_loss'], 'b.')\n",
        "plt.plot(history.history['val_iob_output_loss'], 'b-')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<matplotlib.lines.Line2D at 0x7f27e053a6d8>]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtkAAAEICAYAAACKx+iJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XuclWW5//HPlxnAQDkjKiiYogmV\nphM5W7eRVOIh0TLDdm01Df2paSdTy9x2cJt20HbZwTxmmRqlUZnnUEtUBiNF1ERFhUCOYpqCwPX7\n436Ws1iumVnMrFlrzcz3/Xo9r/Wcn2utmbnXNfdz3/ejiMDMzMzMzMqnV7UDMDMzMzPrbpxkm5mZ\nmZmVmZNsMzMzM7Myc5JtZmZmZlZmTrLNzMzMzMrMSbaZmZmZWZk5ybaqk/SopInVjsNaJukqSd+s\ndhxmZj2JpJmSjq92HNY+TrLtDZIWSlonaVjB+r9JCkljynCNNyVrETE+ImZ29NxFruXCycwsT1bO\nvz9veZSkX0paKekVSQ9KOqTgmMi2vSxphaRfSRpU6jXMeion2VboGeCo3IKkdwD9qhdOzyapvtox\nmFn3JGkI8BdgHTAeGAZcBFwr6YiC3XePiC2BtwKDgXMrGGrFuey1cnCSbYWuAf47b/lo4Of5O0jq\nK+k7kp6T9IKkn0h6S7ZtoqRFkr4gaZmkJZKOzbZNA/4L+FJWI/L7bP0btR7ZuS+W9M9sulhS37bO\nvbkkHZo1U3kxq/HeLW/bGZIWS/qXpCckTcrWT5DUJOml7H1/r4Vz5+L8clbrs1DSf23m53eGpKXA\nlS1c41OSHpO0WtKtkkbnbQtJp0p6Orv+tyX1yrb1knS2pGezz/DnkgbmHbuvpPuyz+V5ScfkXXaw\npD9mn8sDknZqz2dvZjXjc8DLwHERsTQiXo2IXwHnAd+VpMIDIuIlYAYwrj0XlPRpSQskrZI0Q9J2\n2XpJuigrl16S9Iikt2fbDpI0Pyt7Fkv6YgvnPkbSXyX9UNIaSY/nyu9s+0BJl2ffHYslfVNSXcGx\nF0laSZF/IrLy80xJTynV/N+g9I8KksZkZe+07LtrSX6crX23ZdunSJqbvfenJE3Ou/ToLLZ/SbpN\nBXebrXY5ybZC9wMDJO2WFT5TgV8U7PMtYBdgD2BnYCRwTt72bYCB2frjgEskDY6IS4FfAhdGxJYR\n8aEi1/8KsHd27t2BCcDZbZ17c96gpF2AXwGfBYYDNwO/l9RH0q7AKcC7I2Ir4ABgYXbo94HvR8QA\nYCfghlYusw2pVmgk6R+VS7NzQ2mf3xBgNDCtSPxTgC8DH87ivzd7P/kOBxqAPYEpwKey9cdk0/tI\nNVJbAj/Mzjsa+BPwg+y8ewBz8845FfgaqRZrAemL2My6rg8Av4mIjQXrbwB2IJVTm8jK28NI3xWb\nRdL+wPnAkcC2wLPAddnmDwL7ZdccmO2zMtt2OXBCVia/Hbirlcu8B3iKVP7+D/DbXCIMXAWsJ5W7\n78queXzBsU8DIyhevn2G9N7fC2wHrAYuKdjnfcDY7NxnqLnZTIvfbZImkCqzTgcGZZ/Dwrxzfhw4\nFtga6AMU/SfDalBEePJERED6o34/6Q//fGAycDtQDwQwBhDwCrBT3nGNwDPZ/ETgVaA+b/syYO9s\n/irgm8Wum80/BRyUt+0AYGEp5y7yfmYCxxdZ/1XghrzlXsDi7Pw7Z+d8P9C74Lh7SEnmsDY+x4mk\ngrx/3robsuuW8vmtA7Zo5fx/ItU85cf/b2B0thzA5LztJwF3ZvN3AiflbdsVeD37GZ8F3NjCNa8C\nLstbPgh4vNq/s548edq8qaC8XQCcWGSfLbJyZJ9sOYCXgBeBDcDjwMhSrlGw/nJSJUtuecus/BkD\n7A/8g5SI9io47jngBGBAG+/tGOCfgPLWPQh8kpQ4rwXekrftKODPecc+18b5HwMm5S1vm1d+jsk+\np7flbb8QuDybb+277afARS1ccyZwdt7yScAt1f498lTa5JpsK+Ya0n/Ox1DQVIRUw9kPmJM1KXgR\nuCVbn7MyItbnLf+bVJiWYjtS7UbOs9m6cpy76DUi1eI8T/rSWECq4T4XWCbputztTFLN+S7A45Jm\nq6BzUIHVEfFKkfdRyue3PCJea+Xco4Hv5x2/ipS8j8zb5/ki137Te8/m60lfQNuTvghasjRvvj2f\nu5nVlhWkRLHQtnnbc/aMiEGkBPzHwL2SttjM6xWWvS+TaqtHRsRdpLtql5DK3kslDch2/QjpH/tn\nJd0tqbGVayyOLBvN5Mq/0UBvYEle2flTUu1wTn65Wcxo4Ma84x8j/dMxooVztFX25ra57O2mnGTb\nm0TEs6QOkAcBvy3YvIJUmzw+IgZl08BIHWJKOn0b2/9JKshydsjWldMm18jaHW5Pqs0mIq6NiH2z\nfQK4IFv/ZEQcRSqULwCmS+rfwjUGF2zLvY9SPr+2PqPnSbdOB+VNb4mI+/L22b7Itd/03rNt64EX\nsvO6nbVZz3EH8OFcn408R5LKg38UHhARrwOXATuSmm5sjsKytz8wlOay9/8iYi9Se+9dSM0niIjZ\nETGFVPbeROtN9UYWtCXPlX/Pk2qyh+WVmwMiYnz+22sj/ueBAwvK3i0iYnHePptT9ua2ueztppxk\nW0uOA/YvqI3N1fr+DLhI0tYAkkZKOqDE875Aagvckl8BZ0sannXuOIc3twnfHPWStsibepMK6IMl\nTcqWv0AqfO+TtKuk/bMOKa+REuKNAJI+IWl49hm8mJ2/sC1jvq9l7bz/EzgE+HUZPj+AnwBnSRqf\nHT9Q0kcL9jld0mBJ2wOnAddn638FfE7SjpK2BP4XuD67O/BL4P2SjpRUL2mopD02Iy4z61ouIrV/\nvlzSNlkZeRSp/fDpBTXCAGR9dY4llY1Pt3Lu3gVlbz2p/DlW0h5ZGfu/wAMRsVDSuyW9JyuTXyGV\nvxuzMvS/JA3MEvyXaL3c3Ro4VVLvrFzcDbg5IpYAt5E6dA7IOjHuJOm9m/F5/QQ4L+u/QvY9NaVg\nn69K6peVz8eyadnb0nfb5dnnMimLa6Skt21GXFajnGRbURHxVEQ0tbD5DFJbvvslvUSqDdm1hX0L\nXQ6My2633VRk+zeBJuBh4BHgoWxde/2Y9GWQm66MiCeAT5A6+K0APgR8KCLWAX1JHRNXkG7RbU1q\nqwypjfqjkl4mdYKcGhGvtnDdpaROMf8kJa8nRsTj2baOfH5ExI2kmvTrsuPnAQcW7PY7YA6p4+If\nSZ87wBWk5kD3kO5WvEbqzENEPEe6e/EFUhOUuaQOOmbWDUXESmBfUhOQ+aSmG58HPhkR1xfs/ves\n7FtN6sx9eESsauX0N7Np2XtuRNxB6pvyG2AJqfZ2arb/AFIFxGpSU4qVwLezbZ8EFmbl3YmkUapa\n8gCp4+EKUufFI7L3CWnkrD7Ze10NTKd4c5mWfJ80ssptkv5F6vz5noJ97iaV73cC34mI27L1LX63\nRcSDpIT8ImBNdo7RWJenIv+omlkHKD298hcRMapK1w9gbNa+3MysR1AacvT4rLlfpa89hlRx0bug\n35D1YK7JNjMzMzMrMyfZZmZmZmZl5uYiZmZmZmZl5ppsMzMzM7Myq692AOUwbNiwGDNmTLXDMDPb\nbHPmzFkREcPb3rP7cJltZl1ZqeV2t0iyx4wZQ1NTS6PNmZnVLknPtr1X9+Iy28y6slLLbTcXMTMz\nMzMrMyfZZmZmZmZl5iTbzMzMzKzMnGSbmZmZmZWZk2wzMzMzszJzkm1mZmZmVmY9NsmeNQvOPz+9\nmplZbXOZbWZdTbcYJ3tzzZoFkybBunXQpw/ceSc0NlY7KjMzK8Zltpl1RT2yJnvmzFRYb9iQXmfO\nrHZEZmbWEpfZZtYV9cgke+LEVBtSV5deJ06sdkRmZtYSl9lm1hX1yOYijY3pduPMmamw9m1HM7Pa\n5TLbzLqiHplkQyqkXVCbmXUNLrPNrKvpkc1FzMzMzMw6k5NsMzMzM7Myc5JtZmZmZlZmTrLNzMzM\nzMqspCRb0mRJT0haIOnMItv7Sro+2/6ApDF5287K1j8h6YC89YMkTZf0uKTHJDVm64dIul3Sk9nr\n4I6/TTMzMzOzymkzyZZUB1wCHAiMA46SNK5gt+OA1RGxM3ARcEF27DhgKjAemAz8KDsfwPeBWyLi\nbcDuwGPZ+jOBOyNiLHBntmxmZmZm1mWUUpM9AVgQEU9HxDrgOmBKwT5TgKuz+enAJEnK1l8XEWsj\n4hlgATBB0kBgP+BygIhYFxEvFjnX1cBh7XtrZmbWUW3dycz2OVLSfEmPSrq20jGamdWiUpLskcDz\necuLsnVF94mI9cAaYGgrx+4ILAeulPQ3SZdJ6p/tMyIilmTzS4ERpb8dMzMrl1LuZEoaC5wF7BMR\n44HPVjxQM7MaVK2Oj/XAnsCPI+JdwCsUaRYSEQFEsRNImiapSVLT8uXLOzVYM7MeqpQ7mZ8GLomI\n1QARsazCMZqZ1aRSkuzFwPZ5y6OydUX3kVQPDARWtnLsImBRRDyQrZ9OSroBXpC0bXaubYGiBXZE\nXBoRDRHRMHz48BLehpmZbaZS7mTuAuwi6a+S7pc0udiJXDFiZj1NKUn2bGCspB0l9SF1ZJxRsM8M\n4Ohs/gjgrqwWegYwNRt9ZEdgLPBgRCwFnpe0a3bMJGB+kXMdDfyuHe/LzMwqo55Utk8EjgJ+JmlQ\n4U6uGDGznqa+rR0iYr2kU4BbgTrgioh4VNLXgaaImEHqwHiNpAXAKlIiTrbfDaQEej1wckRsyE79\nGeCXWeL+NHBstv5bwA2SjgOeBY4s03s1M7PNU8qdzEXAAxHxOvCMpH+Qku7ZlQnRzKw2tZlkA0TE\nzcDNBevOyZt/DfhoC8eeB5xXZP1coKHI+pWkmm0zM6uuN+5kkpLrqcDHC/a5iVSDfaWkYaTmI09X\nNEozsxrkJz6amVlR2WhRuTuZjwE35O5kSjo02+1WYKWk+cCfgdOzyhIzsx6tpJpsMzPrmUq4kxnA\n57PJzMwyrsk2MzMzMyszJ9lmZmZmZmXmJNvMzMzMrMycZJuZmZmZlZmTbDMzMzOzMnOSbWZmZmZW\nZk6yzczMzMzKzEm2mZmZmVmZOck2MzMzMyszJ9lmZmZmZmXmJNvMzMzMrMycZJuZmZmZlZmTbDMz\nMzOzMnOSbWZmZmZWZk6yzczMzMzKzEm2mZmZmVmZOck2MzMzMyszJ9lmZmZmZmXmJNvMzMzMrMyc\nZJuZmZmZlZmTbDMzMzOzMnOSbWZmZmZWZk6yzczMzMzKzEm2mZmZmVmZlZRkS5os6QlJCySdWWR7\nX0nXZ9sfkDQmb9tZ2fonJB2Qt36hpEckzZXUlLf+XEmLs/VzJR3UsbdoZmbtVUL5f4yk5Xll9vHV\niNPMrNbUt7WDpDrgEuADwCJgtqQZETE/b7fjgNURsbOkqcAFwMckjQOmAuOB7YA7JO0SERuy494X\nESuKXPaiiPhO+9+WmZl1VInlP8D1EXFKxQM0M6thpdRkTwAWRMTTEbEOuA6YUrDPFODqbH46MEmS\nsvXXRcTaiHgGWJCdr8uaNQvOPz+9mpl1c6WU/2ZmVkQpSfZI4Pm85UXZuqL7RMR6YA0wtI1jA7hN\n0hxJ0wrOd4qkhyVdIWlwsaAkTZPUJKlp+fLlJbyNjps1CyZNgq9+Nb060Tazbq6U8h/gI1mZPV3S\n9pUJzcystlWz4+O+EbEncCBwsqT9svU/BnYC9gCWAN8tdnBEXBoRDRHRMHz48IoEPHMmrFsHGzak\n15kzK3JZM7Na9ntgTES8E7id5ruam6hGxYiZWTWVkmQvBvJrJkZl64ruI6keGAisbO3YiMi9LgNu\nJGtGEhEvRMSGiNgI/Iwaal4ycSL06QN1del14sRqR2Rm1qnaLP8jYmVErM0WLwP2KnaialSMmJlV\nUylJ9mxgrKQdJfUhdWScUbDPDODobP4I4K6IiGz91Gz0kR2BscCDkvpL2gpAUn/gg8C8bHnbvPMe\nnltfCxob4c474RvfSK+NjdWOyMysU7VZ/heU2YcCj1UwPjOzmtXm6CIRsV7SKcCtQB1wRUQ8Kunr\nQFNEzAAuB66RtABYRSqIyfa7AZgPrAdOjogNkkYAN6a+kdQD10bELdklL5S0B6nN9kLghPK93Y5r\nbHRybWY9Q4nl/6mSDiWV8auAY6oWsJlZDVGqcO7aGhoaoqmpqe0dzcxqjKQ5EdFQ7TgqyWW2mXVl\npZbbfuKjmZmZmVmZOck2MzMzMyszJ9lmZmZmZmXmJNvMzMzMrMycZJuZmZmZlZmTbDMzMzOzMnOS\nbWZmZmZWZk6yzczMzMzKzEm2mZmZmVmZOck2MzMzMyszJ9lmZmZmZmXmJNvMzMzMrMycZJuZmZmZ\nlZmTbDMzMzOzMnOSbWZmZmZWZk6yzczMzMzKzEm2mZmZmVmZOck2MzMzMyszJ9kVNGsWnH9+ejUz\nMzOz7qu+2gH0FLNmwaRJsG4d9OkDd94JjY3VjsrMzMzMOoNrsitk5syUYG/YkF5nzqx2RGZmZmbW\nWZxkV8jEiakGu64uvU6cWO2IzMzMzKyzuLlIhTQ2piYiM2emBNtNRczMzMy6LyfZFdTY6OTazMzM\nrCdwcxEzM2uRpMmSnpC0QNKZrez3EUkhqaGS8ZmZ1aqSkuy2CllJfSVdn21/QNKYvG1nZeufkHRA\n3vqFkh6RNFdSU976IZJul/Rk9jq4Y2/RzMzaQ1IdcAlwIDAOOErSuCL7bQWcBjxQ2QjNzGpXm0l2\niYXsccDqiNgZuAi4IDt2HDAVGA9MBn6UnS/nfRGxR0Tk13ycCdwZEWOBO7NlMzOrvAnAgoh4OiLW\nAdcBU4rs9w1Suf9aJYMzM6tlpdRkl1LITgGuzuanA5MkKVt/XUSsjYhngAXZ+VqTf66rgcNKiNHM\nzMpvJPB83vKibN0bJO0JbB8Rf6xkYGZmta6UJLvNQjZ/n4hYD6wBhrZxbAC3SZojaVrePiMiYkk2\nvxQYUSwoSdMkNUlqWr58eQlvw8zMyklSL+B7wBdK2Ndltpn1KNXs+LhvROxJaoZysqT9CneIiCAl\n428SEZdGRENENAwfPryTQzUz65EWA9vnLY/K1uVsBbwdmClpIbA3MKNY50eX2WbW05SSZLdVyG6y\nj6R6YCCwsrVjIyL3ugy4keZmJC9I2jY717bAstLfjpmZldFsYKykHSX1IfWxmZHbGBFrImJYRIyJ\niDHA/cChEdFU/HRmZj1HKUl2q4VsZgZwdDZ/BHBXVgs9A5iajT6yIzAWeFBS/6w3OpL6Ax8E5hU5\n19HA79r31szMrCOy5n+nALcCjwE3RMSjkr4u6dDqRmdmVtvafBhNRKyXlCtk64ArcoUs0BQRM4DL\ngWskLQBWkRJxsv1uAOYD64GTI2KDpBHAjalvJPXAtRFxS3bJbwE3SDoOeBY4sozv18zMNkNE3Azc\nXLDunBb2nViJmMzMuoKSnvjYViEbEa8BH23h2POA8wrWPQ3s3sL+K4FJpcTVU8ya5cexm5mZmXUl\nfqx6jZs1CyZNgnXroE8fuPNOJ9pmZmZmtc6PVa9xM2emBHvDhvQ6c2a1IzIzMzOztjjJrnETJ6Ya\n7Lq69DpxYrUjMjMzM7O2uLlIjWtsTE1E3CbbzMzMrOtwkt0FNDY6uTYzMzPrStxcxMzMzMyszJxk\nm5mZmZmVmZNsMzMzM7Myc5JtZmZmZlZmTrLNzMzMzMrMSXY3N2sWnH9+ejUzMzOzyvAQft2YH8lu\nZmZmVh2uye7G/Eh2MzMzs+pwkt2N+ZHsZmZmZtXh5iLdmB/JbmZmZlYdTrK7OT+S3czMzKzy3FzE\nzMzMzKzMnGSbmZmZmZWZk2wzMzMzszJzkm0t8oNszMzMzNrHHR+tKD/IxszMzKz9XJNtRflBNmZm\nZmbt1yOT7Ai45RZ4/PFqR1K7/CAbMzMzs/brkc1F1qyBI4+EAw6AX/+62tHUJj/IxszMzKz9emRN\n9qBBcNppMH06PPxwtaOpXY2NcNZZTrDNejJJkyU9IWmBpDOLbD9R0iOS5kr6i6Rx1YjTzKzWlJRk\nl1DI9pV0fbb9AUlj8radla1/QtIBBcfVSfqbpD/krbtK0jNZgT1X0h7tf3st+/znYcAAOPfczji7\nmVnXJ6kOuAQ4EBgHHFUkib42It4REXsAFwLfq3CYZmY1qc0ku8RC9jhgdUTsDFwEXJAdOw6YCowH\nJgM/ys6XcxrwWJHLnh4Re2TT3M18TyUZPBg+9zm48Ub429864wpmZl3eBGBBRDwdEeuA64Ap+TtE\nxEt5i/2BqGB8ZmY1q5Sa7DYL2Wz56mx+OjBJkrL110XE2oh4BliQnQ9Jo4CDgcs6/jba57OfTU1H\nXJtdXh5f26zbGAk8n7e8KFu3CUknS3qKVJN9arETSZomqUlS0/LlyzslWDOzWlJKkl1KIfvGPhGx\nHlgDDG3j2IuBLwEbi1zzPEkPS7pIUt9iQZWjwB40KDUbmTED5sxp1ymsQG587a9+Nb060Tbr/iLi\nkojYCTgDOLuFfS6NiIaIaBg+fHhlAzQzq4KqdHyUdAiwLCKKpbZnAW8D3g0MIRXab1KuAvu001LT\nkf/5n3afwvJ4fG2zbmUxsH3e8qhsXUuuAw7r1IjMzLqIUpLsUgrZN/aRVA8MBFa2cuw+wKGSFpIK\n5f0l/QIgIpZEsha4kqx5SWcZMAC++EX44x/hwQc780o9g8fXNutWZgNjJe0oqQ+pj82M/B0kjc1b\nPBh4soLxmZnVrFKS7DYL2Wz56Gz+COCuiIhs/dRs9JEdgbHAgxFxVkSMiogx2fnuiohPAEjaNnsV\nqUZkXofeYQk+8xkYOtRts8shN772N77hR7GbdXVZ879TgFtJndRviIhHJX1d0qHZbqdIelTSXODz\nNH8XmJn1aG0+jCYi1kvKFbJ1wBW5QhZoiogZwOXANZIWAKtIiTPZfjcA84H1wMkRsaGNS/5S0nBA\nwFzgxHa+t5JttRWcfjqceWZqQ+zEsGMaG/0ZmnUXEXEzcHPBunPy5k+reFBmZl2AUoVz19bQ0BBN\nTU0dOsfLL8Nb3wrvehfcemuZAjMza4OkORHRUO04KqkcZbaZWbWUWm73yCc+FrPllvClL8Ftt8Ff\n/1rtaHouD/9nZmZm3YGT7DwnnQQjRnikkWrx8H9mZmbWXTjJztOvH5xxRuqwd8891Y6m5/Hwf2bW\nkiuvhEsuqXYUZmalc5Jd4MQTYZttXJtdDR7+z8xacvPNaQSotWurHYmZWWmcZBd4y1vgrLNSLeqf\n/1ztaHoWD/9nZi2ZNg1WrIDf/rbakZiZlcZJdhHTpsF226Xa7G4w+EqX0tiY/slxgm1m+SZNSiNA\nXXpptSMxMyuNk+wittgCvvxluPfeVKNqtc+jkph1b716wac/ne4yPvFEtaMxM2ubk+wWHH88jBrl\n2uyuwKOSmPUMxx4L9fXws59VOxIzs7Y5yW5B377wla/AfffB7bdXOxprjUclMesZRoyAww6Dq66C\n116rdjRmZq1zkt2KT30KdtgBzjnHtdm1zKOSmPUcJ5wAK1fCjTdWOxIzs9Y5yW5Fnz5w9tnwwANw\nyy3VjsZa4lFJzHqO/fdPHSB/+tNqR2Jm1jon2W045hgYM8a12bWuI6OSuNOkWdfRq1caAeruu+Hx\nx0s7xn/jZlYNTrLb0Lt36lDX1AR//GO1o7Fyc6dJs67nmGNK7wDpv3EzqxYn2SX45CfT7UmPNNL9\nuNOkWdczYgQcfnhpHSD9N25m1eIkuwS9e6fmIg89BDNmVDsaKyd3mjTrmk44AVatavsJkP4bN7Nq\nUXSDqtmGhoZoamrq1GusXw/jxkG/finZ7uV/T7qNWbNS7dbEie40aZUnaU5ENFQ7jkoqR5m9cSPs\nsguMHJnaZ7fGf+NmVk6lltv1lQimO6ivT7XZn/wk3HQTfPjD1Y7IyqWxsf0dJv3FbVYduQ6QZ5yR\nOkC+7W0t79vev3Ezs45wfexmOOoo2HXX1Db71VerHY1VkztTmVXfMcek5nyXXlrtSMzM3sxJ9mao\nq0vDQM2bB/vuCwsXVjsiqxZ3pjKrvq23Th0gr77aT4A0s9rjJHszHX44/P738NRTsNdefuR6T+XO\nVGa1Ydq01AHyN7+pdiRmZptykt0OhxySxs3ebjuYPBm+9S0P7dfTdPQpk344hll5vO99sPPOfgKk\nmdUed3xsp513hvvvh+OPT08afPDBNGbrgAHVjswqpSMdJidNSs1M+vTxo+DNOqJXL/j0p1MHyMce\ng912q3ZEZmaJa7I7oH9/uPZa+N730vjZEyakQt6sNW7PbVZe7gBpZrXISXYHSfC5z8Edd8Dq1SnR\nbuvhCNazuT23dSWSJkt6QtICSWcW2f55SfMlPSzpTkmjKx2jO0CaWS1ykl0mEyfCnDkwfjx85COp\nCcmGDdWOympRR9tzm1WKpDrgEuBAYBxwlKRxBbv9DWiIiHcC04ELKxtlcsIJqaJj+vRqXN3M7M1K\nSrJLqMnoK+n6bPsDksbkbTsrW/+EpAMKjquT9DdJf8hbt2N2jgXZOfu0/+1V1qhR6cljJ5yQOkNO\nngwrVlQ7KqtFjY3pH7H2tul2p0mrkAnAgoh4OiLWAdcBU/J3iIg/R8S/s8X7gVEVjhFwB0gzqz1t\nJtkl1mQcB6yOiJ2Bi4ALsmPHAVOB8cBk4EfZ+XJOAwpbMV8AXJSda3V27i6jb1/4yU/gssvg3nvT\nMH9z5lQ7Kusu/BAcq7CRwPN5y4uydS05DvhTsQ2SpklqktS0fPnyMoaYO38azu8vf4H588t+ejOz\nzVZKTXabNRnZ8tXZ/HRgkiRl66+LiLUR8QywIDsfkkYBBwOX5U6SHbN/dg6ycx7WnjdWbccdlwr7\nCNhnnzTyiFlHdaTTpGvArTNJ+gTQAHy72PaIuDQiGiKiYfjw4Z0SgztAmlktKSXJLqUm4419ImI9\nsAYY2saxFwNfAjbmbR8KvJido6VrAZ1fK1IODQ2pFnuffeDYY+Gkk1JiZNZe7e006Rpwa6fFwPZ5\ny6OydZuQ9H7gK8ChEbG2QrG9yfDh8OEPw89/Dq++Wq0ozMySqnR8lHQIsCwi2t2QohK1IuUwfDjc\neiucfjr8+Mdp9JHZs6sdlXV8L/pJAAAaa0lEQVRV7e006WEDrZ1mA2OzvjJ9SM3/ZuTvIOldwE9J\nCfayKsS4CXeANLNaUUqSXUpNxhv7SKoHBgIrWzl2H+BQSQtJzU/2l/SL7JhB2TlaulaXU18PF14I\nN90Ey5fDe94Dp54K//pXtSOzrqg9nSY9bKC1R3ZX8RTgVlL/mRsi4lFJX5d0aLbbt4EtgV9Lmitp\nRgunq4iJE2HsWDcZMbPqKyXJbrMmI1s+Ops/ArgrIiJbPzUbfWRHYCzwYEScFRGjImJMdr67IuIT\n2TF/zs5Bds7fdeD91ZQpU1KHnJNOgh/+MD2Z7Kabqh2V9QR+DLy1V0TcHBG7RMROEXFetu6ciJiR\nzb8/IkZExB7ZdGjrZ+xc+R0gH320mpGYWU/XZpJdYk3G5cBQSQuAzwNnZsc+CtwAzAduAU6OiLZG\njz4D+Hx2rqHZubuNgQNTgn3ffTBkSHqAwuGHw6JF1Y7Murv2Dhvo9tzW1Rx9dLpj87OfVTsSM+vJ\nSmqTXUJNxmsR8dGI2DkiJkTE03nHnpcdt2tEvGlop4iYGRGH5C0/nZ1j5+ycVetE05n23jt1ivzW\nt1Kb7XHj4Ac/8ANsrPa4Pbd1NbkOkFdf7Q6QZlY9fuJjFfXuDWecAfPmpdrFU0+F//gPmDu32pGZ\nNetIe243M7FqmTYNXnwRfv3rakdiZj2Vk+wa8Na3wi23wC9/CQsXpqH/Tj8dXnml2pGZtb89t5uZ\nWDVNnAi77NKxDpD+J9HMOsJJdo2Q4OMfh8ceS2Nqf+c78Pa3w5+KPjvNrLLa057bzUysmnIdIP/6\n1/Z1gPQ/iWbWUU6ya8yQIamzzt13wxZbwEEHwdSpsHRptSMz2zwdHTbQtYjWUbkOkO2pzfY/iWbW\nUU6ya9R++6W22V/7Gtx4I7ztbXDOOWmcbbOuoCPDBroW0cph2DD4yEfa9wRIjy1vZh3lJLuG9e2b\nEuuHH4b3vS8lK6NHw2c+k9pum9W69g4b2JFaRNeAW74TT0wdIA86CBYsKP24jo4tb2bmJLsL2HXX\nVJs9f35qOvLTn8LOO8MnPpEScLPupr21iK4Bt0L77QeXXQZ/+xu84x3p6bvr15d2bHv/STQzAyfZ\nXcpuu8EVV8DTT8Npp6WnRe6+Oxx8MNxzD0RUO0Kz8mhvLWJH29G6Frx7Ou64VElx4IFp2NQJE1LS\nbWbWmZxkd0GjRsF3vwvPPZeSkNmz4b3vTWNs/+53sHFjtSM067j21CJ2dExv14J3X9ttB7/9LUyf\nDv/8J7z73XDmmX5YjZl1HifZXdiQIXD22al99g9/mEYgOeywNPTfVVelmjyznqQj7WjdDrxn+MhH\n0lCpRx8NF1wA73ynRw4xs87hJLsb6NcPTj4ZnnwyPdCmd+801vZOO8H3vgf/+le1IzSrnPa2o3U7\n8J5j8GC4/HK444505+9972t+QmS5+B8vM3OS3Y3U16cH2sydCzffnJLsL3wBRo6Ek07y49rNWlOt\nduBWPZMmwSOPwBe/mJLuceNSX5eO8j9eZgZOsrslKXXwmTkzFe6HHZY6TL7rXfCe96QvEz+y3ezN\nKt0O3KqvXz/49rfhwQdh663h8MPhox/t2APA/I+XmYGT7G5v773Tgxj++U+4+GJ4+WU4/njYdttU\nu/33v1c7QrOuzeMpdw977ZU6kf/v/8Lvf988mlN7Rm3yP15mBqDoBuO+NTQ0RFNTU7XD6BIi4L77\n0ljbN9wAa9em4aymTUtjcPfvX+0IzXoWSXMioqHacVRSrZfZTzwBn/403Htv+qfp+ONTh8mBA0s/\nx6xZqQZ74sTNf9ppe44zs8optdx2kt2DrVoFv/hFSrjnz4ettkoPuDnhhDT+tpl1PifZtWnjxvQQ\nm+98J3Uq32ILOPTQVEZOnpw6mJdbri33unWpBtx3RsxqU6nltpuL9GBDhsCpp8K8efCXv6S2iFde\nCXvskdpuX3EFvPRStaM0M6u8Xr3SHb4nnoD770+12XfdlRLt7baDU06BBx4o70PA3JbbrHtxkm1I\nsM8+cPXVsHgxfP/7qe32ccfBiBFwxBHpAQ5+aIOZ9TRSqnT4wQ9S35bf/x723z/Vcu+9N+y6K3z9\n6+lJvB3lttxm3Yubi1hREamW5le/Sm23ly6FLbdMI5UcdRR84AOdc7vUrKdxc5Guac0a+M1v4Jpr\nmmuc/+M/4JOfhCOPTHcK28Ntss1qn9tkW9ls2AB3350S7t/8BlavTl8gRxyREu7//M9U82Jmm89J\ndtf33HNw7bUp4Z4/P1VAHHxwSrYPOmjzOkx2hBN0s8pwkm2dYt06uO22lHD/7ndpvO1tt4WPfSyN\nTjJhQrq9amalcZLdfUSkh3794hcp6V66NCXc+++f7gJOmZLKy87gTpNmleOOj9Yp+vSBQw5Jj29f\ntgyuvz61S/zRj9LrTjvBl78Mc+akGnAzs55CSg/9+u53U/+Wv/4VPvtZeOop+H//L3WYbGyECy+E\nf/yjvNd2p0mz2uMk29qtX790O/S3v00J95VXwi67pC+QhgYYPDgNdXXeeam5iTtOmnU9kiZLekLS\nAklnFtm+n6SHJK2XdEQ1YqxFvXqlNtq5hHrevPTAotdfhzPOSB0mx4+Hr3wFmpo6PkqJO02a1R43\nF7GyW7YM7rgjDQt4773pywXSbdOGhtSGe99904gm7e0cZNZd1HJzEUl1wD+ADwCLgNnAURExP2+f\nMcAA4IvAjIiY3tZ5e3qZ/dxzqbndjTfCPfek2udRo1KTksMOg/32a1/Hcj8Ax6wy3Cbbasbq1em2\naS7pnj071eZAqsnZd9/mxHv06OrGalZpNZ5kNwLnRsQB2fJZABFxfpF9rwL+4CR786xcCX/4A9x0\nE9x6a7rjN2BAKhP32w/e+17Yc8/OG83JbbnNNl+p5XZ9iSebDHwfqAMui4hvFWzvC/wc2AtYCXws\nIhZm284CjgM2AKdGxK2StgDuAfpmMUyPiP/J9r8KeC+wJjv9MRExt5Q4rTYNHpzacR9ySFp+9dWU\naOeS7l/9Kj11EmD77dMXyjveAe98Z3rdeWeoL+k31czKbCTwfN7yIuA97TmRpGnANIAddtih45F1\nE0OHwtFHp+nf/04dy//0p1TD/cc/pn36909NT9773pR4T5gAffuW5/rF2nI7yTYrjzZTl+x24SXk\n3S6UNCP/diEpiV4dETtLmgpcAHxM0jhgKjAe2A64Q9IuwFpg/4h4WVJv4C+S/hQR92fnO72U2hDr\nmt7ylvRFsd9+aXnDhtSk5N57U4333LnpgQ8bN6btffvCuHEp4c5N73wnbLONRzIx6yoi4lLgUkg1\n2VUOpyb169fcZATghRdSsn3PPalfy9lnp/VbbJE6mudquvfeOx3bHrm23Lma7M1ty+2mJmYtK6V+\ncAKwICKeBpB0HTAFyE+ypwDnZvPTgR9KUrb+uohYCzwjaQEwISJmAS9n+/fOJhe6PVRdHey+e5pO\nOSWte+01eOwxeOSRND38MNx+O/z8583HDR26aeKdO0e5anjMjMXA9nnLo7J1VgEjRsBHP5omgFWr\nUmXE3XenxPub30xPm+zdG9797pRwv/OdsOOOMGYMbL112xURjY2piUh723K7qYlZy0pJsku5XfjG\nPhGxXtIaYGi2/v6CY0fCGzXkc4CdgUsi4oG8/c6TdA5wJ3BmlqRvwrceu7cttkhDYb3rXZuuX7ly\n08T7kUfgiivSeN2QCvo990w1O7lphx1c423WTrOBsZJ2JCXXU4GPVzeknmvIkDTW9pQpaXnNGrjv\nvpR03303fPvbsH598/79+qVke8yY5sQ7/3Xw4FQ2Nja2LznuaFMT14Jbd1e1lq4RsQHYQ9Ig4EZJ\nb4+IecBZwFKgD+nW4hnA14sc71uPPdDQoalAzr+luXEjLFwIDz2UHgV///3wk5/AxRen7dtss2nS\n3dCQ2jiaWeuySpNTgFtJfXKuiIhHJX0daIqIGZLeDdwIDAY+JOlrETG+imH3GAMHwoEHpglSm+5n\nnmmeFi5sfr3vPnjxxU2PHzCgOeEePTrVfA8fvum09dYwaFDxioqONDVxLbj1BKUk2aXcLszts0hS\nPTCQ1AGyzWMj4kVJfwYmA/MiYkm2aa2kK0nDQpm1qFcveOtb03RENkrv66+nmu7772+ebropbaur\nS81L8hPvsWPTecxsUxFxM3Bzwbpz8uZnk8p2q7J+/dKITeNb+BfnxRc3Tbxzr089BX/+M7z0UvHj\n6uth2LDiCfgpp6QnW37wg6lcLVVHasFdA25dRSlJdim3C2cARwOzgCOAuyIiJM0ArpX0PVLHx7HA\ng5KGA69nCfZbSJ0qLwCQtG1ELMnadB8GzOvwu7Qep3dv2GuvNJ18clq3YkVzTff996enVv7kJ2lb\n377paZU77ZRGM8lNO+2Uang8uomZdXWDBsEee6SpmNdeS+Xk8uVpWraseT5/euihtG3NmuZjr7km\nvQ4dmmrG85ul5ObHjElNAaH9teAdrQF3gl7cxo3w5JNp5K8nn0yVVu94B+y2WxqswNqnzdShlNuF\nwOXANVnHxlWkRJxsvxtInSTXAydHxAZJ2wJXZ+2yewE3RMQfskv+MkvCBcwFTiznG7aea9gwOPjg\nNEGqQXn88ZRwP/ZYqs1ZsCA9SCf/6ZT19enLIZd05yfgO+7Y/KVhZtaVbbFFeijOqBLvS6xbl5Ly\nRYs2rR1/5hn4+9/TA3fWrdv0mG23bU6+jzgClixJTwp+8ME0slSfPqnSo0+f4vPXXgtr16akcO3a\nNMb4uHHN+7V2R9JNVJKI9ECk2bPT00Zzr8XuZPTqlb7v8gcZeMc7UhJeV9exGF58Md0FWbo0/R71\n75/6HQwZkv5ZGzSoY9dYvz6de/Hi9Du6ePGm84sWwemnwwkntP8abfHDaMwKRKSCf8GCNOWS79xU\nWBANGwbbbZe+PLbb7s3z222X2oV31sMkrGur5YfRdBaX2T3Dxo2pLM1PvvObqTz//KYdNcuhvj4l\n3LmkOzfft28anWVxXoPV3XaDt799033yp/zjFy1KMe+/Pxx0UCrjO5IAVtKyZSmRzp+WL0/bevdO\nI9K8+93N0y67pJ9RbpCBefPS61NPpe9HSLXb48enhPvtb29OvgcMSENP5pLn1qbCf8CKGTRo08Q7\nN5+/LBVPpJcubR4KOKdPHxg5Mk2jRsEnPtFc8bY5/MRHs04QkUY4ySXfTz8N//xnmpYsSa9Ll6Za\n8kLDh2+agI8a1XwLdcyYtOxEvOdxkm09VURzm+y1azd9bW3dvHmpz81OO6UHmK1du+m+LS2/8EJq\nMrhxY0rMdtopJcqF+61d2/xU4pbU16cye4cd0jR69Jvny93BfsOGNJLWyy+nKX++cN2//pXu1M6e\nnf6ZgfSex41rTqYbGjZv2NtXXoH585uT79y0bFnrx0np+2+bbVqehg1LHXdXrUrTypXN88XWrV7d\nnPDnDByYfib5SXT+68iR6TrlGG3MSbZZlWzYkGoJckl3YRKem5Yu3bSQ6NUrFQL5iffo0c3z22+f\n/gu37sVJtlnllNome+PG5uT7wgvhW99K63r1gg99KCWrzz3XPC1a9ObKlSFDUhnev39qvz50aKrp\n3bAh1eDnpvzl/Plc4hyR4shvxtiWurqUdO65Zxp9pqEhzW+5ZXs+tdYtW9Zc2/3KK6kiKT+BHj68\n/P2aNm5MzU1WrUrzI0dWdtQwJ9lmNW7dulTDsHBhmp59tnl+4cJ0yyv/VpeUasB32CEVnv36pUKl\nf//S54cMSYVeV7nN2RM4yTarbaW05d6wIVWe5Cfezz6b2qbPmpUSZSm1bx4wICWd9fWpLC42/+KL\n6YFDucR+6tTUjGPLLVN5vuWWLU+PPJIS69df3/y25+4YWppSy22PmWBWJX36NI9oUszrrze3A8yf\nnn8+dRL5979TrUH+aynq69N//blbmzvskGrJ85cHDizPezQz6+pKeSpmXV0qR7ffHvbZp3n9+een\nJiobNqRk+dhj4ayz2r7m+eenJDtXDzp+fGnHQUqUX39984dHdMfQ8nOSbVajevduHv6qFBHpdmJh\n4p3/umJFStJzNS333QfXX//mzkcDBmyaeOdGG/j3v9M1Xn110/nC5fz5iOZOKkOHpjZxufmWpkGD\nPG65mdWO9j4Vs71DFXbkQT/tPdZP8Cw/J9lm3YSUmoT067d5x23YkDoE5RLv/CQ8N8zTihVp3/r6\n1Ku8X7/0mpv69YOttkoPp8jflotl5crm6ZFHmjuwFOsgCinBHjIk3R5taEjTXnvBrru6qYuZdR2l\n1IKX87iOHOsneJafk2yzHq6urnmowb33Lr7P2rUp8S3n6CcbN6bhEFes2DQJz03Ll6fONJddBv/3\nf+mY/v1T55299mpOvHfZxbXeZla72lsL3t7j2ntsRxJ714IX5yTbzNpU6hBPm6NXr9QsZNCg1Bmo\nJbmHBs2Zkx6Y0NQEP/0pXHxx2r7VVpsm3g0NqZ27E28zs81T6WYx0LFa8FpPzp1km1lNq6tLnX7G\nj4f//u+0bv369JTO/MT7Rz9Kj4WG9OS6rbYqPspKWyOwFDaFyW/6kr/sZitmZkk1asE72kSlEgm6\nk2wz63Lq65ufMHbMMWnd66+nByXMmZNeX3mlecp1/FyypHk+tz6XmG+u3r3fnIj/4hepVt3MrKep\ndC14R5qoVKoNuZNsM+sWevdOTy/bfffNO27DhuZRWXKJd/6oKaVMuWM640EPZmbdWTU6ana0DXmp\nnGSbWY9WV9f8EAczM6u8SnfU7EiCvjmcZJuZmZlZl9ORkVvam6BvDifZZmZmZtajdGSIxFJ5kCsz\nMzMzszJzkm1mZmZmVmZOss3MzMzMysxJtpmZmZlZmTnJNjMzMzMrMyfZZmZmZmZlpoiodgwdJmk5\n8Gw7Dh0GrChzOB1Ra/FA7cXkeNpWazHVWjxQWzGNjojh1Q6iklxmd6pai6nW4oHai8nxtK3WYiqp\n3O4WSXZ7SWqKiIZqx5FTa/FA7cXkeNpWazHVWjxQmzFZ22rt51Zr8UDtxVRr8UDtxeR42laLMZXC\nzUXMzMzMzMrMSbaZmZmZWZn19CT70moHUKDW4oHai8nxtK3WYqq1eKA2Y7K21drPrdbigdqLqdbi\ngdqLyfG0rRZjalOPbpNtZmZmZtYZenpNtpmZmZlZ2TnJNjMzMzMrsx6RZEuaLOkJSQsknVlke19J\n12fbH5A0phNj2V7SnyXNl/SopNOK7DNR0hpJc7PpnM6KJ7veQkmPZNdqKrJdkv4v+3welrRnJ8ez\na957nyvpJUmfLdinUz8jSVdIWiZpXt66IZJul/Rk9jq4hWOPzvZ5UtLRnRzTtyU9nv1cbpQ0qIVj\nW/0ZlzGecyUtzvu5HNTCsa3+TZY5puvz4lkoaW4Lx5b9M7L2cZldUlw1U27XQpmdXaOmym2X2e2O\nqfuU2RHRrSegDngKeCvQB/g7MK5gn5OAn2TzU4HrOzGebYE9s/mtgH8UiWci8IcKfkYLgWGtbD8I\n+BMgYG/ggQr//JaSBn6v2GcE7AfsCczLW3chcGY2fyZwQZHjhgBPZ6+Ds/nBnRjTB4H6bP6CYjGV\n8jMuYzznAl8s4Wfa6t9kOWMq2P5d4JxKfUae2vUzdJldWlw1WW5Xq8zOrlFT5bbL7PbFVLC9S5fZ\nPaEmewKwICKejoh1wHXAlIJ9pgBXZ/PTgUmS1BnBRMSSiHgom/8X8BgwsjOuVUZTgJ9Hcj8wSNK2\nFbr2JOCpiGjP0+HaLSLuAVYVrM7/PbkaOKzIoQcAt0fEqohYDdwOTO6smCLitohYny3eD4wqx7Xa\nG0+JSvmbLHtM2d/0kcCvynEt6zQus8ujWuV2VcpsqL1y22V2x2LqDmV2T0iyRwLP5y0v4s0F5Bv7\nZL/8a4ChnR1YdovzXcADRTY3Svq7pD9JGt/JoQRwm6Q5kqYV2V7KZ9hZptLyH1glPyOAERGxJJtf\nCowosk81P6tPkWquimnrZ1xOp2S3Qq9o4dZstT6j/wReiIgnW9heyc/IWuYyuzS1Wm7XUpkNtV1u\nu8xuXZcvs3tCkl2TJG0J/Ab4bES8VLD5IdKttt2BHwA3dXI4+0bEnsCBwMmS9uvk65VEUh/gUODX\nRTZX+jPaRKR7VTUz/qWkrwDrgV+2sEulfsY/BnYC9gCWkG711YqjaL1GpCb/Dqw21FiZDTX4+1rL\nZTbUVrntMrskXb7M7glJ9mJg+7zlUdm6ovtIqgcGAis7KyBJvUmF9S8j4reF2yPipYh4OZu/Gegt\naVhnxRMRi7PXZcCNpFtD+Ur5DDvDgcBDEfFC4YZKf0aZF3K3W7PXZUX2qfhnJekY4BDgv7IvkTcp\n4WdcFhHxQkRsiIiNwM9auE41PqN64MPA9S3tU6nPyNrkMrsENVpu11qZDTVYbrvMblt3KbN7QpI9\nGxgracfsv+ypwIyCfWYAud7ERwB3tfSL31FZG6PLgcci4nst7LNNrn2hpAmkn1OnfIFI6i9pq9w8\nqVPGvILdZgD/rWRvYE3e7bfO1OJ/sZX8jPLk/54cDfyuyD63Ah+UNDi77fbBbF2nkDQZ+BJwaET8\nu4V9SvkZlyue/Dafh7dwnVL+Jsvt/cDjEbGo2MZKfkbWJpfZbcdUq+V2rZXZUGPltsvsknWPMrvU\nHpJdeSL1sv4HqXfsV7J1Xyf9kgNsQbq9tQB4EHhrJ8ayL+l21cPA3Gw6CDgRODHb5xTgUVIP3vuB\n/+jEeN6aXefv2TVzn09+PAIuyT6/R4CGCvzM+pMK4IF56yr2GZG+KJYAr5Panx1HavN5J/AkcAcw\nJNu3Abgs79hPZb9LC4BjOzmmBaS2crnfpdyIC9sBN7f2M+6keK7JfkceJhXC2xbGky2/6W+ys2LK\n1l+V+93J27fTPyNP7f45usxuPaaaK7epcpmdXaOmyu0W4nGZ3UZM2fqr6AZlth+rbmZmZmZWZj2h\nuYiZmZmZWUU5yTYzMzMzKzMn2WZmZmZmZeYk28zMzMyszJxkm5mZmZmVmZNsMzMzM7Myc5JtZmZm\nZlZm/x8PiRAvow2oPwAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 864x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Vj3wSVIVrsRo",
        "colab_type": "text"
      },
      "source": [
        "## Simple Entity Disambiguator"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VvsLH3wYDhY6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from gensim.utils import tokenize\n",
        "from scipy.spatial import distance\n",
        "\n",
        "\n",
        "def predict_entities(doc):\n",
        "  \n",
        "  global ft, model, ent_men_emb, id2wikiid, wikiid2name\n",
        "  \n",
        "  tokens = tokenize(doc, deacc=True)\n",
        "  word_embs = []\n",
        "  \n",
        "  for token in tokens:\n",
        "    if token in ft:\n",
        "      word_embs.append(ft[token])\n",
        "    else:\n",
        "      word_embs.append(ft['unk'])\n",
        "      \n",
        "  word_embs = np.array(word_embs)\n",
        "\n",
        "  iobs, mentions = model.predict(word_embs.reshape(1, -1, 300), batch_size=1)\n",
        "  iobs = np.argmax(iobs.reshape(-1, 3)[:word_embs.shape[0]], axis=-1)\n",
        "  mentions = mentions.reshape(-1, 300)\n",
        "    \n",
        "  best = {}\n",
        "  \n",
        "  start, end = -1, -1\n",
        "  \n",
        "  for i, iob in enumerate(iobs):\n",
        "    \n",
        "    if iob == 1:\n",
        "      start = i\n",
        "      \n",
        "    if iob in [1, 2] and (len(iobs) <= i+1 or iobs[i+1] in [0, 1]):\n",
        "      \n",
        "      end = i\n",
        "      \n",
        "      mention = mentions[i]\n",
        "      \n",
        "      distances = distance.cdist([mention], ent_men_emb[1:], 'cosine')\n",
        "      eid = 1 + np.argmin(distances)\n",
        "      wikiid = id2wikiid[eid]\n",
        "      name = wikiid2name[wikiid]\n",
        "      \n",
        "      best[f'{start}:{end}'] = wikiid\n",
        "  \n",
        "  return best"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RV8mfletEEXn",
        "colab_type": "code",
        "outputId": "eba8aba3-647a-43b9-b885-d263c68c6938",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "predict_entities('Ali daei scored 104-th goal for Bayern Munich in barcelona')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'0:1': 196804, '6:8': 172326, '9:9': 68187}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vi_5tLZkXCno",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ft.similar_by_vector(ent_men_emb[wikiid2id[name2wikiid['FC Barcelona']]])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8CCaZvFq-HFR",
        "colab_type": "text"
      },
      "source": [
        "# 3. Evaluate Model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WeJ3CEtLkxSg",
        "colab_type": "text"
      },
      "source": [
        "## Load Saved Mention Detector"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G8a31JJTpf75",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import os\n",
        "from keras.models import load_model\n",
        "\n",
        "if not os.path.exists('el_mention_model.h5'):\n",
        "  ! cp drive/My\\ Drive/e2e_el/el_mention_model.h5 el_mention_model.h5\n",
        "\n",
        "model = load_model('el_mention_model.h5')#, custom_objects={'f1': f1})"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "he99KsvOk5ZH",
        "colab_type": "text"
      },
      "source": [
        "## Test Model With Arbitary Input Text"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2q54kWy_lZZ0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def text_to_entity_names(text, human_readable=True):\n",
        "  \n",
        "  clean = [w for w in text.split(' ')]\n",
        "#   word_ids = np.array([word2id[w] for w in clean])\n",
        "  \n",
        "  disambiguated = predict_entities(text)\n",
        "\n",
        "  if not human_readable:\n",
        "    return disambiguated\n",
        "  \n",
        "  human_readable = {}\n",
        "  \n",
        "  for boundary, wikiid in disambiguated.items():\n",
        "    start, end = tuple(map(int, boundary.split(':')))\n",
        "    mention = ' '.join([f'[{boundary:5s}]'] + clean[start:end+1])\n",
        "    human_readable[mention] = wikiid2name[wikiid]\n",
        "  \n",
        "  return human_readable"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VGHLtvcMmI4f",
        "colab_type": "code",
        "outputId": "dfa8e6cf-d767-4d91-b406-bf9197d8c65b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "from_testa = 'They were held up by a gritty 84 from Paul Johnson but ex-England fast bowler Martin McCague took four for 55 .'\n",
        "politics = 'President Barack Obama last week said he will impose a blanket tariff on Mexican imports from June 10 to try to pressure Mexico to tackle large flows of mostly Central American migrants passing through en route to the United States'\n",
        "sports = 'Barcelona plays with Sevilla'\n",
        "\n",
        "text_to_entity_names(sports)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'[0:0  ] Barcelona': 'FC Barcelona', '[3:3  ] Sevilla': 'Sevilla FC'}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ErJwnTSslIo1",
        "colab_type": "text"
      },
      "source": [
        "## Functions to evaluate Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5MPfcRO08Eeh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def count_correct(predicted, gold, strong_match=True, return_totals=True):\n",
        "  \n",
        "  total_pred, total_gold = len(predicted), len(gold)\n",
        "  correct = 0\n",
        "  \n",
        "  if strong_match:\n",
        "    for b in predicted.keys():\n",
        "      if b in gold and predicted[b] == gold[b]:\n",
        "        correct += 1\n",
        "  else:\n",
        "    for b, e in gold.items():\n",
        "      for pb, pe in predicted.items():\n",
        "        if pe == e:\n",
        "          g_start, g_end = int(b.split(':')[0]), int(b.split(':')[1])\n",
        "          p_start, p_end = int(pb.split(':')[0]), int(pb.split(':')[1])\n",
        "          if p_start in range(g_start, g_end+1) or p_end in range(g_start, g_end+1):\n",
        "            correct += 1\n",
        "\n",
        "  if return_totals:\n",
        "    return correct, total_pred, total_gold\n",
        "  else:\n",
        "    return correct"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yakmAuuyzNLQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from commons import show_progress\n",
        "\n",
        "reload(process_aida)\n",
        "\n",
        "def calculate_micro_f1(aida_path='aida-testa.tsv', total_docs=216, strong_match=True):\n",
        "  \n",
        "  total_pred = 0       # number of entities in predictions\n",
        "  total_gold = 0       # number of gold entities in test set\n",
        "  correct = 0          # number of predicted entities that match entity\n",
        "                       # and boundary (in strong match)\n",
        "  doc_idx = 0\n",
        "\n",
        "  for tokens, golds in process_aida.gen_tokens_with_golds(aida_path):\n",
        "\n",
        "    show_progress(percent=doc_idx/total_docs, title=f'{correct} correct, {total_pred} pred, {total_gold} total ')\n",
        "    \n",
        "    predicted = predict_entities(' '.join(tokens))\n",
        "    corr, tot_pred, tot_gold = count_correct(predicted, golds,\n",
        "                                            strong_match=strong_match)\n",
        "    total_pred += tot_pred\n",
        "    correct += corr\n",
        "    total_gold += tot_gold\n",
        "    \n",
        "    doc_idx += 1\n",
        "  \n",
        "  show_progress(percent=1.0, done=True, done_message=f' {correct} correct, {total_pred} pred, {total_gold} total')\n",
        "  \n",
        "  precision, recall = correct/total_pred, correct/total_gold\n",
        "  f_score = (2 * precision * recall) / (precision + recall)\n",
        "  \n",
        "  return precision, recall, f_score"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fCsLFnixPCB0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "for tokens, golds in process_aida.gen_tokens_with_golds('aida-testa.tsv'):\n",
        "  print(' '.join(tokens))\n",
        "  print(predict_entities(' '.join(tokens)))\n",
        "  print(golds)\n",
        "  break"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BFxIFuWBj8lX",
        "colab_type": "text"
      },
      "source": [
        "## Micro F1 Scores for Baseline"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fhly7f1Ojhz8",
        "colab_type": "code",
        "outputId": "043fe116-e7b8-4f46-f37c-52d60af23ea1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 357
        }
      },
      "source": [
        "precision, recall, f_score = calculate_micro_f1(strong_match=False)\n",
        "print(f'evaluation on aida validation set with weak matching:')\n",
        "print(f'precision: {precision:.3f}\\nrecall:    {recall:.3f}\\nf1-score:  {f_score:.3f}')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0 correct, 87 pred, 77 total [>--------------------------------------------------] 0.93%  "
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-27-a1a94d5f4f09>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mprecision\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrecall\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mf_score\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcalculate_micro_f1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstrong_match\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf'evaluation on aida validation set with weak matching:'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf'precision: {precision:.3f}\\nrecall:    {recall:.3f}\\nf1-score:  {f_score:.3f}'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-26-c397f18a37ec>\u001b[0m in \u001b[0;36mcalculate_micro_f1\u001b[0;34m(aida_path, total_docs, strong_match)\u001b[0m\n\u001b[1;32m     15\u001b[0m     \u001b[0mshow_progress\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpercent\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdoc_idx\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0mtotal_docs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtitle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34mf'{correct} correct, {total_pred} pred, {total_gold} total '\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 17\u001b[0;31m     \u001b[0mpredicted\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpredict_entities\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m' '\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtokens\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     18\u001b[0m     corr, tot_pred, tot_gold = count_correct(predicted, golds,\n\u001b[1;32m     19\u001b[0m                                             strong_match=strong_match)\n",
            "\u001b[0;32m<ipython-input-14-08b69ff030f0>\u001b[0m in \u001b[0;36mpredict_entities\u001b[0;34m(doc)\u001b[0m\n\u001b[1;32m     37\u001b[0m       \u001b[0mmention\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmentions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     38\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 39\u001b[0;31m       \u001b[0mdistances\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdistance\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcdist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmention\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ment_men_emb\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'cosine'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     40\u001b[0m       \u001b[0meid\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m1\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margmin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdistances\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     41\u001b[0m       \u001b[0mwikiid\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mid2wikiid\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0meid\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/scipy/spatial/distance.py\u001b[0m in \u001b[0;36mcdist\u001b[0;34m(XA, XB, metric, *args, **kwargs)\u001b[0m\n\u001b[1;32m   2778\u001b[0m             cdist_fn = getattr(_distance_wrap,\n\u001b[1;32m   2779\u001b[0m                                \"cdist_%s_%s_wrap\" % (metric_name, typ))\n\u001b[0;32m-> 2780\u001b[0;31m             \u001b[0mcdist_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mXA\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mXB\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2781\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mdm\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2782\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IkWjaH6FkC_N",
        "colab_type": "code",
        "outputId": "af9414c2-e3b5-4b6c-edca-f7939d22858a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        }
      },
      "source": [
        "precision, recall, f_score = calculate_micro_f1(strong_match=True)\n",
        "print(f'evaluation on aida validation set with strong matching:')\n",
        "print(f'precision: {precision:.3f}\\nrecall:    {recall:.3f}\\nf1-score:  {f_score:.3f}')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[===================================================] 100% done!\n",
            "evaluation on aida validation set with strong matching:\n",
            "precision: 0.524\n",
            "recall:    0.686\n",
            "f1-score:  0.594\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B0qBtwzjjn0L",
        "colab_type": "code",
        "outputId": "579cb5d9-ae17-4724-c552-413abdea35fb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        }
      },
      "source": [
        "precision, recall, f_score = calculate_micro_f1(aida_path='aida-testb.tsv',\n",
        "                                                strong_match=False)\n",
        "print(f'evaluation on aida test set with weak matching:')\n",
        "print(f'precision: {precision:.3f}\\nrecall:    {recall:.3f}\\nf1-score:  {f_score:.3f}')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[===================================================] 100% done!\n",
            "evaluation on aida test set with weak matching:\n",
            "precision: 0.448\n",
            "recall:    0.619\n",
            "f1-score:  0.520\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OkqU-RWbjoXR",
        "colab_type": "code",
        "outputId": "fef8ea8d-6858-4e77-f72b-9531a8590701",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        }
      },
      "source": [
        "precision, recall, f_score = calculate_micro_f1(aida_path='aida-testb.tsv',\n",
        "                                                strong_match=True)\n",
        "print(f'evaluation on aida test set with strong matching:')\n",
        "print(f'precision: {precision:.3f}\\nrecall:    {recall:.3f}\\nf1-score:  {f_score:.3f}')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[===================================================] 100% done!\n",
            "evaluation on aida test set with strong matching:\n",
            "precision: 0.415\n",
            "recall:    0.573\n",
            "f1-score:  0.481\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LQbEnly9BsXV",
        "colab_type": "text"
      },
      "source": [
        "# 4. Extras"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0hsw_c2XRi0w",
        "colab_type": "code",
        "outputId": "9b20f514-db24-4589-e475-b9d3d830d79f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 544
        }
      },
      "source": [
        "import sys\n",
        "\n",
        "ram = 0\n",
        "\n",
        "for var, obj in locals().items():\n",
        "  if sys.getsizeof(obj) > 10**3:\n",
        "    print(f'{var:16s}: {sys.getsizeof(obj) / 10**6}')\n",
        "  ram += sys.getsizeof(obj)\n",
        "\n",
        "print(f'used ram: {ram / 10**6}')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "FastText        : 0.002\n",
            "name2wikiid     : 41.943144\n",
            "wikiid2name     : 41.943144\n",
            "id2wikiid       : 41.943144\n",
            "wikiid2id       : 41.943144\n",
            "wikiid2mentions : 20.971616\n",
            "_i13            : 0.002271\n",
            "_i14            : 0.001155\n",
            "_i17            : 0.001067\n",
            "tokens          : 0.027336\n",
            "golds           : 0.00228\n",
            "_i21            : 0.001155\n",
            "_i32            : 0.001087\n",
            "Model           : 0.003096\n",
            "LSTM            : 0.002\n",
            "GRU             : 0.001056\n",
            "Embedding       : 0.001056\n",
            "Dense           : 0.002\n",
            "TimeDistributed : 0.001464\n",
            "Dropout         : 0.001056\n",
            "Bidirectional   : 0.002\n",
            "Concatenate     : 0.001056\n",
            "Lambda          : 0.001464\n",
            "_i34            : 0.001087\n",
            "_i36            : 0.001087\n",
            "_i38            : 0.001055\n",
            "_i39            : 0.001055\n",
            "_i41            : 0.001055\n",
            "_i43            : 0.001055\n",
            "_i44            : 0.001083\n",
            "used ram: 188.827673\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zA7z_GlSgK88",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}