blog_tds_fd905cb22df7_bert_embeddings_wsd.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "blog_tds_fd905cb22df7_bert_embeddings_wsd.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyMan6CefH8/x0AitIry2fkL",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "6c0dbf0d4ae24283840657a55918543e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_5da3a3ac31bf4cae8d47a96ad6950ad4",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_4950e62ffb4f454fb4d0191c1e0cd7fc",
              "IPY_MODEL_be5edd0d62464921b917d7bcdf0cda5e"
            ]
          }
        },
        "5da3a3ac31bf4cae8d47a96ad6950ad4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4950e62ffb4f454fb4d0191c1e0cd7fc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_8de07f880ba948768cfd753b2da3380f",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 213450,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 213450,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_21b504974045464dba293f9a7fbdce95"
          }
        },
        "be5edd0d62464921b917d7bcdf0cda5e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_0ffb099cd42946e99a5a1e317952c9d9",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 213k/213k [00:00&lt;00:00, 1.22MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c3f6506955f04621975e4d7549990a83"
          }
        },
        "8de07f880ba948768cfd753b2da3380f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "21b504974045464dba293f9a7fbdce95": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0ffb099cd42946e99a5a1e317952c9d9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c3f6506955f04621975e4d7549990a83": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f07c86c999f44e2eb2f7242c082b9e52": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_e839673f672848e9800a191c12c27986",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_70544e3f3ef44fdd96d83cb3eb09af1d",
              "IPY_MODEL_b2998b88deb14169bc112e35705160fc"
            ]
          }
        },
        "e839673f672848e9800a191c12c27986": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "70544e3f3ef44fdd96d83cb3eb09af1d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_dd7b264075e648d68f4ac5d5a2baaa57",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 433,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 433,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6f66076952a24b6a8587e90e8519ae94"
          }
        },
        "b2998b88deb14169bc112e35705160fc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_16dfa848ad7b446e9ff4f76f25e01e82",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 433/433 [00:00&lt;00:00, 492B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c365dc81177c4a97bf696c34a9b7f489"
          }
        },
        "dd7b264075e648d68f4ac5d5a2baaa57": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6f66076952a24b6a8587e90e8519ae94": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "16dfa848ad7b446e9ff4f76f25e01e82": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c365dc81177c4a97bf696c34a9b7f489": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fb67e3b0c65e40969dd8bd1b117f6605": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_ff84320a8cbd4ca79520f928b54354ca",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_aa72789613944aadac10bb15ef079937",
              "IPY_MODEL_ed9fa31e02fb4632ab97c73e1c098d19"
            ]
          }
        },
        "ff84320a8cbd4ca79520f928b54354ca": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "aa72789613944aadac10bb15ef079937": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_1b856941c65b4fb29eda8959f3cc2fb7",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 435779157,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 435779157,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7eb17b7daa574a92bcbfd673e2bea6d1"
          }
        },
        "ed9fa31e02fb4632ab97c73e1c098d19": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_201e8c73d5dd4a1f8a3d0b64ff385246",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 436M/436M [00:11&lt;00:00, 37.9MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_48b2da00c883499886d4d80ba9f096c2"
          }
        },
        "1b856941c65b4fb29eda8959f3cc2fb7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7eb17b7daa574a92bcbfd673e2bea6d1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "201e8c73d5dd4a1f8a3d0b64ff385246": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "48b2da00c883499886d4d80ba9f096c2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/sujitpal/nlp-deeplearning-ai-examples/blob/master/blog_tds_fd905cb22df7_bert_embeddings_wsd.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "m2jrbccUMkEp"
      },
      "source": [
        "# WSD using BERT Raw Embeddings\n",
        "\n",
        "This notebook explores an idea proposed by Ajit Rakasekharan in his blog post [Examining BERT raw embeddings](https://towardsdatascience.com/examining-berts-raw-embeddings-fd905cb22df7). It is well-known that transformers yield \"contextual\" embeddings for words, unlike word-embeddings. The blog post proposes the idea that it should be possible to disambiguate [homonyms/homographs](https://www.insider.com/words-spelled-same-different-meanings-2019-1), i.e., words that sound and spelled the same but mean different things depending on context, based on their location and proximity to other words in the embedding space.\n",
        "\n",
        "We use the [HuggingFace BERTModel](https://huggingface.co/transformers/model_doc/bert.html#bertmodel) with weights from a `bert-base-cased` pre-trained model for our experiment.\n",
        "\n",
        "For a given homonym/homograph, we will identify several sentences that mention these in different contexts. We will then capture their \"before\" and \"after\" embeddings, i.e. the embeddings for the word from the first and last hidden states of the model, and then compare it (cosine similarity) with all other words in the model vocabulary.\n",
        "\n",
        "We expect to observe, for each distinct context:\n",
        "* different distributions of similarity values, and\n",
        "* different distribution of similar words.\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rjP1dsFlNDb4"
      },
      "source": [
        "## Environment setup"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZJEWj6WnMQoV",
        "outputId": "5ae8f55b-be9f-4866-90a0-ecb42ce3e638"
      },
      "source": [
        "!pip install transformers"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/99/84/7bc03215279f603125d844bf81c3fb3f2d50fe8e511546eb4897e4be2067/transformers-4.0.0-py3-none-any.whl (1.4MB)\n",
            "\u001b[K     |████████████████████████████████| 1.4MB 6.8MB/s \n",
            "\u001b[?25hCollecting tokenizers==0.9.4\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/0f/1c/e789a8b12e28be5bc1ce2156cf87cb522b379be9cadc7ad8091a4cc107c4/tokenizers-0.9.4-cp36-cp36m-manylinux2010_x86_64.whl (2.9MB)\n",
            "\u001b[K     |████████████████████████████████| 2.9MB 34.7MB/s \n",
            "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers) (20.4)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.41.1)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
            "\u001b[K     |████████████████████████████████| 890kB 38.7MB/s \n",
            "\u001b[?25hRequirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.8)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.18.5)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (1.15.0)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.11.8)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (0.17.0)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893257 sha256=f4a6a59338fb05de99192c1848f840deb373299b831848eef3c3755223e624d5\n",
            "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: tokenizers, sacremoses, transformers\n",
            "Successfully installed sacremoses-0.0.43 tokenizers-0.9.4 transformers-4.0.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uTGKzg5VNF9A"
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import torch\n",
        "\n",
        "from transformers import BertModel, BertTokenizer\n",
        "\n",
        "%matplotlib inline"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9VDP-20DNLbB"
      },
      "source": [
        "## Load pre-trained model\n",
        "\n",
        "We choose the pre-trained `bert-base-cased` model because the blog post reports good results with it."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 164,
          "referenced_widgets": [
            "6c0dbf0d4ae24283840657a55918543e",
            "5da3a3ac31bf4cae8d47a96ad6950ad4",
            "4950e62ffb4f454fb4d0191c1e0cd7fc",
            "be5edd0d62464921b917d7bcdf0cda5e",
            "8de07f880ba948768cfd753b2da3380f",
            "21b504974045464dba293f9a7fbdce95",
            "0ffb099cd42946e99a5a1e317952c9d9",
            "c3f6506955f04621975e4d7549990a83",
            "f07c86c999f44e2eb2f7242c082b9e52",
            "e839673f672848e9800a191c12c27986",
            "70544e3f3ef44fdd96d83cb3eb09af1d",
            "b2998b88deb14169bc112e35705160fc",
            "dd7b264075e648d68f4ac5d5a2baaa57",
            "6f66076952a24b6a8587e90e8519ae94",
            "16dfa848ad7b446e9ff4f76f25e01e82",
            "c365dc81177c4a97bf696c34a9b7f489",
            "fb67e3b0c65e40969dd8bd1b117f6605",
            "ff84320a8cbd4ca79520f928b54354ca",
            "aa72789613944aadac10bb15ef079937",
            "ed9fa31e02fb4632ab97c73e1c098d19",
            "1b856941c65b4fb29eda8959f3cc2fb7",
            "7eb17b7daa574a92bcbfd673e2bea6d1",
            "201e8c73d5dd4a1f8a3d0b64ff385246",
            "48b2da00c883499886d4d80ba9f096c2"
          ]
        },
        "id": "XpJbvP5INI--",
        "outputId": "d03fad66-8bef-4587-f7a3-6a30afeca01f"
      },
      "source": [
        "tokenizer = BertTokenizer.from_pretrained('bert-base-cased')\n",
        "model = BertModel.from_pretrained('bert-base-cased', return_dict=True)"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "6c0dbf0d4ae24283840657a55918543e",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=213450.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "f07c86c999f44e2eb2f7242c082b9e52",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=433.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "fb67e3b0c65e40969dd8bd1b117f6605",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=435779157.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zjVcBZ6kOxSR"
      },
      "source": [
        "### Hidden States\n",
        "\n",
        "We want our model to return the hidden states, so we can capture them. The `output_hidden_states` parameter needs to be set to True for this.\n",
        "\n",
        "There are 13 hidden states, the first corresponding to the embedding (projecting the vocabulary word to the (768,) embedding) and the rest corresponding to the 12 layers in the BERT-base model.\n",
        "\n",
        "Output of each hidden state is `(batch_size, number_of_tokens, 768)`."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OwfiA06rM8Tb"
      },
      "source": [
        "text = \"Conan went to a prison cell.\"\n",
        "inputs = tokenizer(text, return_tensors=\"pt\")\n",
        "outputs = model(**inputs, output_hidden_states=True)"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "871mgj67Ojug",
        "outputId": "b4025f91-4546-4a88-9254-c11cda5abb1b"
      },
      "source": [
        "outputs"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "BaseModelOutputWithPoolingAndCrossAttentions([('last_hidden_state',\n",
              "                                               tensor([[[ 0.3345, -0.0393,  0.0458,  ...,  0.0241,  0.0804,  0.0365],\n",
              "                                                        [ 0.0278, -0.5531,  0.5972,  ...,  0.2188, -0.1919,  0.0281],\n",
              "                                                        [ 0.2310,  0.1144,  0.2451,  ...,  0.4046, -0.1479,  0.1663],\n",
              "                                                        ...,\n",
              "                                                        [ 0.1710,  0.1697,  0.0063,  ...,  0.2594,  0.0258,  0.0875],\n",
              "                                                        [ 0.2277, -0.3697,  0.1023,  ...,  0.3099, -0.0642, -0.1183],\n",
              "                                                        [ 0.8567,  0.1250, -0.6334,  ...,  0.4787,  0.1390, -0.1556]]],\n",
              "                                                      grad_fn=<NativeLayerNormBackward>)),\n",
              "                                              ('pooler_output',\n",
              "                                               tensor([[-0.6734,  0.4065,  0.9997, -0.9870,  0.9466,  0.7674,  0.9722, -0.9835,\n",
              "                                                        -0.9595, -0.6332,  0.9572,  0.9962, -0.9946, -0.9995,  0.6838, -0.9575,\n",
              "                                                         0.9805, -0.4738, -0.9999, -0.7750, -0.5400, -0.9997,  0.2727,  0.9519,\n",
              "                                                         0.9491,  0.1065,  0.9760,  0.9999,  0.8799, -0.1746,  0.2028, -0.9783,\n",
              "                                                         0.8528, -0.9981,  0.0876,  0.0903,  0.5676, -0.2806,  0.8968, -0.7827,\n",
              "                                                        -0.6409, -0.8559,  0.4211, -0.4749,  0.8017,  0.1081,  0.1746, -0.1369,\n",
              "                                                        -0.1652,  0.9997, -0.9378,  0.9964, -0.9832,  0.9855,  0.9880,  0.2840,\n",
              "                                                         0.9914,  0.1336, -0.9950,  0.2907,  0.9243,  0.1870,  0.9064, -0.2188,\n",
              "                                                         0.2990, -0.1878, -0.8112,  0.1303, -0.4573,  0.3859,  0.4116,  0.2606,\n",
              "                                                         0.9698, -0.8549,  0.0248, -0.8536,  0.2902, -0.9998,  0.9349,  0.9999,\n",
              "                                                         0.7350, -0.9993,  0.9872, -0.2651, -0.6787,  0.3796, -0.9971, -0.9989,\n",
              "                                                         0.0361, -0.5940,  0.7620, -0.9770,  0.5229, -0.8947,  0.9999, -0.5634,\n",
              "                                                        -0.1308,  0.3196,  0.8091, -0.6043, -0.6294,  0.8882,  0.9960, -0.9904,\n",
              "                                                         0.9962,  0.5451, -0.9020, -0.7002,  0.6572,  0.0299,  0.9750, -0.9796,\n",
              "                                                        -0.8383,  0.0206,  0.9301, -0.8521,  0.9722,  0.8447, -0.2683,  0.9999,\n",
              "                                                        -0.1180,  0.9159,  0.9969,  0.7363, -0.7684, -0.1359, -0.7195,  0.8186,\n",
              "                                                        -0.4061, -0.3097,  0.7241, -0.9881, -0.9932,  0.9986, -0.2857,  0.9999,\n",
              "                                                        -0.9984,  0.9779, -0.9999, -0.7860, -0.6539, -0.0896, -0.9591,  0.0265,\n",
              "                                                         0.9867,  0.0375, -0.6291, -0.6563,  0.4010, -0.8000,  0.5571,  0.6708,\n",
              "                                                        -0.9270,  0.9902,  0.9944,  0.9020,  0.9648,  0.1756, -0.8903,  0.8807,\n",
              "                                                         0.9644, -0.9988,  0.9000, -0.9867,  0.9985,  0.9443,  0.8170, -0.9896,\n",
              "                                                         0.9997, -0.6212, -0.1330,  0.1535, -0.1200, -0.9973,  0.3586,  0.3636,\n",
              "                                                         0.7260,  0.9992, -0.9869,  0.9991,  0.9600,  0.1794,  0.7179,  0.9946,\n",
              "                                                        -0.9926, -0.9648, -0.9805,  0.5138,  0.5952,  0.7750,  0.1990,  0.9449,\n",
              "                                                         0.9963,  0.5186, -0.9970, -0.3314,  0.9595, -0.1258,  0.9999, -0.3996,\n",
              "                                                        -0.9994, -0.6702,  0.8076,  0.9659, -0.2272,  0.9539, -0.5987, -0.2538,\n",
              "                                                         0.9712, -0.9927,  0.9950,  0.2232,  0.5373,  0.7946,  0.9838, -0.7073,\n",
              "                                                        -0.0914,  0.1955, -0.5703,  0.9997, -0.9990, -0.1382,  0.4610, -0.9926,\n",
              "                                                        -0.9956,  0.9562,  0.0415, -0.7655, -0.2632,  0.2847,  0.2042,  0.8277,\n",
              "                                                         0.9827, -0.3841, -0.4212, -0.9995, -0.9920, -0.8328, -0.9213, -0.0468,\n",
              "                                                         0.5886, -0.3426, -0.9114, -0.9955,  0.9271,  0.8072, -0.8224, -0.1157,\n",
              "                                                        -0.4004, -0.9950,  0.6679, -0.8350, -0.9972,  0.9989, -0.8047,  0.9778,\n",
              "                                                         0.9232, -0.9912,  0.8100, -0.9964, -0.1703, -0.9928,  0.3421,  0.6272,\n",
              "                                                        -0.6877,  0.0299,  0.9847, -0.9513, -0.5598,  0.4460, -0.9997,  0.8946,\n",
              "                                                        -0.2855,  0.9982,  0.7524, -0.0478,  0.9632,  0.8665, -0.9772, -0.9996,\n",
              "                                                         0.8838,  0.9754, -0.9913, -0.2163,  0.9998, -0.9959, -0.7296, -0.9141,\n",
              "                                                        -0.9839, -0.9992,  0.2162, -0.7483,  0.2277,  0.9481,  0.2372,  0.2199,\n",
              "                                                         0.9918,  0.9851,  0.1369, -0.1099, -0.0119, -0.9559, -0.9825,  0.5544,\n",
              "                                                         0.1700, -0.9999,  0.9996, -0.9866,  0.9939,  0.7631, -0.9889,  0.8520,\n",
              "                                                         0.0544, -0.9372, -0.1288,  0.9997,  0.9709, -0.0063,  0.1028,  0.7809,\n",
              "                                                        -0.0788,  0.6036, -0.8020, -0.4971,  0.1985, -0.9152,  0.9727,  0.5464,\n",
              "                                                        -0.9839,  0.9857, -0.0037,  0.5892, -0.7999,  0.8244,  0.9768, -0.1068,\n",
              "                                                        -0.4011, -0.0896, -0.7495, -0.9559,  0.1019, -0.9936, -0.2747,  0.9397,\n",
              "                                                         0.9696, -0.9741,  0.9830, -0.1123,  0.8770, -0.9961,  0.9999, -0.9919,\n",
              "                                                         0.0548,  0.6263, -0.7639, -0.1318,  0.9796,  0.9713,  0.9475, -0.8551,\n",
              "                                                        -0.6839,  0.8837,  0.9390, -0.9540, -0.0522, -0.9978, -0.4081,  0.9938,\n",
              "                                                         0.9956,  0.0524, -0.3043, -0.9934,  0.9468, -0.8097, -0.7890, -0.0845,\n",
              "                                                        -0.7540,  0.5925,  0.9939, -0.3720,  0.6832,  0.1300, -0.9668,  0.7490,\n",
              "                                                         0.7606,  0.9996, -0.9637,  0.3893,  0.9727, -0.1888, -0.6867,  0.5955,\n",
              "                                                         0.9968, -0.9627, -0.1773, -0.9993, -0.0711, -0.7348, -0.1267, -0.6352,\n",
              "                                                         0.0021, -0.7929,  0.9432,  0.1874,  0.7337, -0.3656,  0.9482, -0.1080,\n",
              "                                                        -0.0309, -0.3035, -0.1199,  0.4507,  0.2712,  0.9637, -0.9493,  0.9996,\n",
              "                                                        -0.3945, -0.9999, -0.9931, -0.7700, -0.9994,  0.4490, -0.9894,  0.9778,\n",
              "                                                         0.7207, -0.9941, -0.9978, -0.9923, -0.9914,  0.8414,  0.5123,  0.0717,\n",
              "                                                         0.4022,  0.6880,  0.0224, -0.0446, -0.0949, -0.9179, -0.5904, -0.9954,\n",
              "                                                         0.6315, -0.9998, -0.6517,  0.9946, -0.9905, -0.8640, -0.9176, -0.8079,\n",
              "                                                        -0.8213,  0.3864,  0.9772,  0.0407, -0.5305, -0.9992,  0.9712, -0.6432,\n",
              "                                                         0.1198, -0.7859, -0.9594,  0.9994,  0.7921, -0.1820, -0.0237, -0.9984,\n",
              "                                                         0.9563, -0.8243, -0.8597, -0.9663,  0.0975, -0.8999, -0.9996, -0.1424,\n",
              "                                                         0.9902,  0.9868,  0.9724,  0.3958, -0.2938, -0.9296,  0.0397, -0.9999,\n",
              "                                                         0.7687,  0.7693, -0.9638, -0.7322,  0.9788,  0.9608, -0.8904, -0.9492,\n",
              "                                                         0.7676,  0.3556,  0.9518, -0.6440, -0.4269,  0.3300, -0.0926, -0.9841,\n",
              "                                                        -0.8918,  0.9920, -0.9967,  0.9316,  0.9897,  0.9966, -0.1918, -0.0017,\n",
              "                                                        -0.9502, -0.9870, -0.6748,  0.3115, -0.9998,  0.9998, -0.9999,  0.6062,\n",
              "                                                        -0.6069,  0.8045,  0.9821, -0.2597, -0.9998, -0.9996,  0.2377,  0.0290,\n",
              "                                                         0.9824,  0.2433,  0.1662, -0.5196, -0.2351,  0.9934, -0.8341, -0.6872,\n",
              "                                                        -0.9941,  0.9991,  0.4458, -0.9968,  0.9843, -0.9988,  0.8350,  0.9501,\n",
              "                                                         0.8557,  0.9584, -0.9975,  0.9999, -0.9996,  0.9921, -0.9999, -0.9980,\n",
              "                                                         0.9996, -0.9845, -0.5941, -0.9994, -0.9936,  0.4915,  0.1023, -0.5628,\n",
              "                                                         0.9800, -0.9996, -0.9971,  0.0359, -0.8209, -0.6964,  0.9875, -0.4843,\n",
              "                                                         0.9726, -0.1618,  0.9554,  0.2034,  0.9935,  0.9930, -0.6814, -0.7202,\n",
              "                                                        -0.9883,  0.9682, -0.4931,  0.2457,  0.9337,  0.1231, -0.7642,  0.3285,\n",
              "                                                        -0.9955,  0.3368, -0.2973,  0.9231,  0.8568,  0.8340, -0.0669, -0.5432,\n",
              "                                                        -0.2026, -0.9814,  0.4266, -0.9991,  0.9660, -0.8701, -0.0468, -0.3554,\n",
              "                                                         0.2913, -0.9365,  0.9992,  0.9974, -0.9957, -0.0181,  0.9722, -0.5337,\n",
              "                                                         0.9657, -0.9865,  0.0145,  0.9195, -0.7195,  0.9683,  0.3960, -0.0449,\n",
              "                                                         0.9406, -0.9900, -0.8326, -0.6437,  0.3049,  0.0634, -0.9545,  0.0118,\n",
              "                                                         0.9763, -0.2210, -0.9992,  0.7830, -0.9982, -0.0580,  0.9635, -0.2513,\n",
              "                                                         0.9998, -0.6513,  0.0653,  0.1714, -0.9994, -0.9972,  0.0551, -0.0937,\n",
              "                                                        -0.8825,  0.9972, -0.1130,  0.7421, -0.9997,  0.3059,  0.9896,  0.2549,\n",
              "                                                         0.8706, -0.5486, -0.9414, -0.8847, -0.4125,  0.0266,  0.8216, -0.9607,\n",
              "                                                        -0.8296, -0.8459,  0.9999, -0.9955, -0.8437, -0.9823,  0.2594,  0.7353,\n",
              "                                                         0.3750,  0.1026, -0.7203,  0.8833, -0.7973,  0.9940, -0.9871, -0.9930,\n",
              "                                                         0.9995,  0.7404, -0.9885, -0.0514, -0.1466,  0.2744,  0.0894,  0.7402,\n",
              "                                                        -0.9196, -0.1264, -0.9936,  0.8369, -0.7520, -0.9729, -0.4964, -0.4270,\n",
              "                                                        -0.9946,  0.9861,  0.9654,  0.9999, -0.9995,  0.6728,  0.0351,  0.9986,\n",
              "                                                         0.0177, -0.5904,  0.7782,  0.9993, -0.6252,  0.8644, -0.1192,  0.0162,\n",
              "                                                         0.3669, -0.5791,  0.9925, -0.9079,  0.0861, -0.9634, -0.9998,  0.9998,\n",
              "                                                         0.0061,  0.9800,  0.2390,  0.8835, -0.7965,  0.9564, -0.9536, -0.8318,\n",
              "                                                        -0.9999,  0.1246, -0.9960, -0.9868,  0.0818,  0.9667, -0.9992, -0.9725,\n",
              "                                                        -0.4669, -0.9999,  0.9132, -0.9689, -0.7224, -0.9727,  0.9946, -0.2288,\n",
              "                                                        -0.5358,  0.9569, -0.9545,  0.9221,  0.9287,  0.4309,  0.1639,  0.1468,\n",
              "                                                        -0.7314, -0.9881, -0.8943, -0.9582,  0.6611, -0.9758, -0.8490,  0.9928,\n",
              "                                                         0.9705, -0.9989, -0.9911,  0.9860,  0.0276,  0.9825, -0.6112, -0.9996,\n",
              "                                                        -0.9997,  0.1759, -0.1383,  0.9919, -0.3844,  0.9887,  0.7159, -0.2942,\n",
              "                                                         0.3744, -0.5238, -0.1674, -0.4480, -0.1897,  0.9999, -0.6482,  0.9851]],\n",
              "                                                      grad_fn=<TanhBackward>)),\n",
              "                                              ('hidden_states',\n",
              "                                               (tensor([[[ 0.4496,  0.0977, -0.2074,  ...,  0.0578,  0.0406, -0.0951],\n",
              "                                                         [-0.1837,  0.3166, -1.7455,  ...,  0.5885,  0.2477,  1.2088],\n",
              "                                                         [ 1.1099,  0.5537,  0.3794,  ...,  0.2915, -0.5682, -0.0233],\n",
              "                                                         ...,\n",
              "                                                         [-0.6015,  1.1529,  0.1755,  ...,  0.3637, -0.1127, -0.2389],\n",
              "                                                         [-0.4275,  0.6968,  0.8957,  ...,  0.1489,  0.6208,  0.2714],\n",
              "                                                         [-0.3162,  0.1007,  0.1413,  ...,  0.5393, -0.4997,  0.3309]]],\n",
              "                                                       grad_fn=<NativeLayerNormBackward>),\n",
              "                                                tensor([[[ 3.3053e-01, -2.6983e-04, -1.2951e-01,  ...,  2.2150e-02,\n",
              "                                                           1.8972e-02, -1.4160e-01],\n",
              "                                                         [-2.1083e-01, -4.3614e-01, -1.4692e+00,  ...,  7.2092e-01,\n",
              "                                                           3.5266e-01,  1.1314e+00],\n",
              "                                                         [ 1.1707e+00,  5.4385e-02,  3.7111e-01,  ...,  2.1252e-01,\n",
              "                                                          -4.3882e-01, -4.2672e-01],\n",
              "                                                         ...,\n",
              "                                                         [-3.9700e-01,  1.6302e+00,  6.2525e-02,  ...,  5.1865e-01,\n",
              "                                                           5.4845e-01, -2.0075e-01],\n",
              "                                                         [ 1.5333e-01,  1.9607e-01,  3.1149e-01,  ..., -4.6332e-02,\n",
              "                                                           3.1949e-01,  1.2494e-02],\n",
              "                                                         [-1.0343e-01, -2.1211e-01,  1.6135e-01,  ...,  6.0379e-01,\n",
              "                                                          -6.3054e-01,  2.8535e-01]]], grad_fn=<NativeLayerNormBackward>),\n",
              "                                                tensor([[[ 0.4479,  0.0140, -0.0806,  ...,  0.0146,  0.0383, -0.2985],\n",
              "                                                         [-0.3506, -0.4397, -0.7414,  ...,  1.2593,  0.2585,  0.5858],\n",
              "                                                         [ 1.2435,  0.0181,  0.3848,  ...,  0.1184, -0.8152, -0.5562],\n",
              "                                                         ...,\n",
              "                                                         [-0.0504,  1.5425, -0.0276,  ...,  0.8404,  0.0271, -0.1918],\n",
              "                                                         [ 0.4584, -0.2571,  0.1990,  ..., -0.3910,  0.0833,  0.0464],\n",
              "                                                         [-0.0116, -0.1475,  0.0590,  ...,  0.2339, -0.2249,  0.0487]]],\n",
              "                                                       grad_fn=<NativeLayerNormBackward>),\n",
              "                                                tensor([[[ 3.3749e-01,  2.6934e-01, -9.8438e-02,  ..., -5.3312e-02,\n",
              "                                                           6.9857e-02,  2.9994e-01],\n",
              "                                                         [-5.0812e-01, -1.7518e-01, -3.1043e-01,  ...,  1.1064e+00,\n",
              "                                                          -1.5990e-01,  2.2664e-01],\n",
              "                                                         [ 1.3895e+00, -1.9637e-01,  1.7965e-01,  ...,  4.5273e-01,\n",
              "                                                          -6.5226e-01, -6.8714e-01],\n",
              "                                                         ...,\n",
              "                                                         [ 1.8780e-01,  1.4723e+00,  5.7000e-01,  ...,  7.3392e-01,\n",
              "                                                           9.8799e-02, -1.0910e-01],\n",
              "                                                         [ 5.8653e-01, -1.2382e-01,  2.1950e-01,  ..., -1.7523e-01,\n",
              "                                                          -6.9208e-02,  2.3663e-01],\n",
              "                                                         [ 1.5334e-02, -9.7833e-02,  2.6790e-02,  ..., -6.3552e-04,\n",
              "                                                          -9.9815e-02,  5.1288e-02]]], grad_fn=<NativeLayerNormBackward>),\n",
              "                                                tensor([[[ 0.3645,  0.4444, -0.2262,  ..., -0.0349,  0.0220,  0.2190],\n",
              "                                                         [-0.4521, -0.7437, -0.4070,  ...,  1.0968, -0.3995,  0.3676],\n",
              "                                                         [ 1.0909,  0.0319,  0.1427,  ...,  0.6974, -0.8083, -0.4766],\n",
              "                                                         ...,\n",
              "                                                         [ 0.4022,  1.4146,  0.6646,  ...,  0.8990,  0.4768,  0.0723],\n",
              "                                                         [ 0.3456, -0.3381,  0.2888,  ..., -0.0721, -0.1104,  0.3590],\n",
              "                                                         [ 0.0257, -0.0524, -0.0128,  ...,  0.0091, -0.0777, -0.0242]]],\n",
              "                                                       grad_fn=<NativeLayerNormBackward>),\n",
              "                                                tensor([[[ 0.3247,  0.1260, -0.3731,  ...,  0.0714,  0.0968,  0.0239],\n",
              "                                                         [-0.4794, -1.0842,  0.0779,  ...,  1.2713, -0.6021,  0.2934],\n",
              "                                                         [ 0.9754, -0.3071,  0.2351,  ...,  0.8232, -0.6971, -0.4601],\n",
              "                                                         ...,\n",
              "                                                         [ 0.3563,  0.9338,  0.3779,  ...,  0.8629,  0.3042,  0.1476],\n",
              "                                                         [ 0.6502, -0.5081,  0.2816,  ...,  0.1204, -0.4086,  0.4505],\n",
              "                                                         [-0.0156, -0.0512,  0.0221,  ..., -0.0174, -0.0547, -0.0404]]],\n",
              "                                                       grad_fn=<NativeLayerNormBackward>),\n",
              "                                                tensor([[[ 0.3341,  0.0094, -0.1456,  ...,  0.0431,  0.0880,  0.1722],\n",
              "                                                         [-0.4843, -1.2115, -0.1514,  ...,  1.2413, -0.8852,  0.7651],\n",
              "                                                         [ 0.4602, -0.2866,  0.0069,  ...,  0.8050, -0.7715,  0.0950],\n",
              "                                                         ...,\n",
              "                                                         [ 0.1899,  0.7636,  0.5243,  ...,  0.9725,  0.2551, -0.1949],\n",
              "                                                         [ 0.4803, -0.4034,  0.2634,  ...,  0.2153, -0.3825,  0.6833],\n",
              "                                                         [-0.0210, -0.0501,  0.0176,  ..., -0.0178, -0.0242, -0.0831]]],\n",
              "                                                       grad_fn=<NativeLayerNormBackward>),\n",
              "                                                tensor([[[ 2.9374e-01, -1.4677e-01, -3.4253e-01,  ..., -1.4169e-01,\n",
              "                                                           9.8878e-02,  8.1031e-02],\n",
              "                                                         [-4.4012e-01, -1.2271e+00,  3.8615e-01,  ...,  8.7152e-01,\n",
              "                                                          -9.8657e-01,  5.6582e-01],\n",
              "                                                         [ 6.3535e-01, -8.3348e-02,  2.7561e-01,  ...,  6.4695e-01,\n",
              "                                                          -3.2603e-01, -3.2781e-02],\n",
              "                                                         ...,\n",
              "                                                         [ 2.6316e-01,  6.7287e-01,  5.0721e-01,  ...,  1.0261e+00,\n",
              "                                                           5.4840e-02, -2.7243e-02],\n",
              "                                                         [ 5.9721e-01, -6.7560e-01,  2.8532e-01,  ..., -1.9241e-01,\n",
              "                                                          -3.8799e-01,  6.0590e-01],\n",
              "                                                         [-3.7614e-02, -1.7720e-02,  4.6684e-02,  ..., -2.2425e-04,\n",
              "                                                          -5.1696e-02, -9.2683e-02]]], grad_fn=<NativeLayerNormBackward>),\n",
              "                                                tensor([[[ 0.0492,  0.0788, -0.2669,  ..., -0.3705,  0.0101,  0.2130],\n",
              "                                                         [-0.4007, -1.4590,  0.4980,  ...,  0.9864, -0.8772,  0.5600],\n",
              "                                                         [ 0.5443, -0.0798,  0.3897,  ...,  0.4159, -0.0654,  0.2073],\n",
              "                                                         ...,\n",
              "                                                         [ 0.3027,  0.2372,  0.7653,  ...,  0.7683, -0.1517,  0.0885],\n",
              "                                                         [ 0.5083, -0.7061,  0.4554,  ..., -0.0489, -0.4520,  0.5562],\n",
              "                                                         [ 0.0034,  0.0532,  0.0299,  ...,  0.0126, -0.0063, -0.0982]]],\n",
              "                                                       grad_fn=<NativeLayerNormBackward>),\n",
              "                                                tensor([[[ 0.0711, -0.0641, -0.3546,  ..., -0.3039,  0.3913,  0.3941],\n",
              "                                                         [-0.5675, -1.5668,  0.4311,  ...,  1.3727, -0.9859,  0.5095],\n",
              "                                                         [ 0.4755, -0.5323,  0.6243,  ...,  0.6161, -0.2810,  0.6452],\n",
              "                                                         ...,\n",
              "                                                         [ 0.2675,  0.3120,  0.0786,  ...,  0.4458, -0.2986, -0.2639],\n",
              "                                                         [ 0.5145, -1.5217,  0.2757,  ...,  0.4696, -0.7561,  0.3427],\n",
              "                                                         [-0.0192,  0.0800,  0.0288,  ...,  0.0343, -0.0100, -0.0966]]],\n",
              "                                                       grad_fn=<NativeLayerNormBackward>),\n",
              "                                                tensor([[[ 0.0952, -0.2272, -0.3143,  ..., -0.1852,  0.4637,  0.3299],\n",
              "                                                         [-0.7109, -1.4428,  0.4245,  ...,  1.1530, -0.6691,  0.4351],\n",
              "                                                         [ 0.5019, -0.4061,  0.5933,  ...,  0.7385, -0.0187,  0.8554],\n",
              "                                                         ...,\n",
              "                                                         [ 0.0845,  0.6305,  0.2612,  ...,  0.5393, -0.0057, -0.0064],\n",
              "                                                         [ 0.4697, -1.2221,  0.4540,  ...,  0.2181, -0.3805,  0.1850],\n",
              "                                                         [-0.0459,  0.0640,  0.0450,  ..., -0.0192, -0.0190, -0.0678]]],\n",
              "                                                       grad_fn=<NativeLayerNormBackward>),\n",
              "                                                tensor([[[ 0.0602, -0.2831, -0.0806,  ..., -0.0423,  0.2015, -0.0201],\n",
              "                                                         [-0.6314, -1.2200,  0.7676,  ...,  0.8817, -0.6027, -0.1099],\n",
              "                                                         [ 0.2940, -0.0517,  0.8078,  ...,  0.5489, -0.1207,  0.4320],\n",
              "                                                         ...,\n",
              "                                                         [ 0.4212,  0.7286,  0.3514,  ...,  0.5829, -0.2635, -0.1941],\n",
              "                                                         [ 0.0910, -1.0270,  0.3477,  ...,  0.3406, -0.2961, -0.2753],\n",
              "                                                         [-0.0624,  0.0609,  0.0394,  ...,  0.0061, -0.0067, -0.0574]]],\n",
              "                                                       grad_fn=<NativeLayerNormBackward>),\n",
              "                                                tensor([[[ 0.3345, -0.0393,  0.0458,  ...,  0.0241,  0.0804,  0.0365],\n",
              "                                                         [ 0.0278, -0.5531,  0.5972,  ...,  0.2188, -0.1919,  0.0281],\n",
              "                                                         [ 0.2310,  0.1144,  0.2451,  ...,  0.4046, -0.1479,  0.1663],\n",
              "                                                         ...,\n",
              "                                                         [ 0.1710,  0.1697,  0.0063,  ...,  0.2594,  0.0258,  0.0875],\n",
              "                                                         [ 0.2277, -0.3697,  0.1023,  ...,  0.3099, -0.0642, -0.1183],\n",
              "                                                         [ 0.8567,  0.1250, -0.6334,  ...,  0.4787,  0.1390, -0.1556]]],\n",
              "                                                       grad_fn=<NativeLayerNormBackward>)))])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "c8P2giakOkse",
        "outputId": "494ad5cf-39d7-43ec-b6c9-ba4e2247a58b"
      },
      "source": [
        "len(outputs.hidden_states)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "13"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "M__3HrbPOplu",
        "outputId": "0ca2ea71-74e1-4667-863d-05625a57b384"
      },
      "source": [
        "# first hidden layer\n",
        "outputs.hidden_states[0].size()"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([1, 9, 768])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hl4AuiO0Ouws",
        "outputId": "2abbb3bf-1404-4836-8f7e-25df7c6ceeb5"
      },
      "source": [
        "# last hidden layer\n",
        "outputs.hidden_states[-1].size()"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([1, 9, 768])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tKVnnxaFPBMl"
      },
      "source": [
        "### Vocabulary and Embeddings\n",
        "\n",
        "We want to compare the input and output embeddings for the homonym/homograph word against all words in the vocabulary.\n",
        "\n",
        "We can find the number of words in the vocabulary, and for each position in this range, we can call `tokenizer.convert_ids_to_tokens` to find the equivalent token value.\n",
        "\n",
        "The embedding for each word (without context) is given by the embedding layer in the model. The base embedding corresponding to a given vocabulary word is the just the row identified by its ID."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ApSOBV5TQD-b",
        "outputId": "370adf39-acef-47e6-effb-cc686bc57df6"
      },
      "source": [
        "vocab_size = model.config.vocab_size\n",
        "vocab_size"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "28996"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "oH8J2X3aQLu3",
        "outputId": "4c7c16c4-7c14-4ac5-a51e-a25857f0abf6"
      },
      "source": [
        "tokenizer.convert_ids_to_tokens(np.random.randint(1000, high=vocab_size))"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'Foley'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "C2GYPt7SQglo",
        "outputId": "77c5f454-131e-4d3a-e330-f88f754af29d"
      },
      "source": [
        "embedding = model.embeddings.word_embeddings\n",
        "E = embedding.weight\n",
        "E.size()"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([28996, 768])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "q4HdSUjQIekM"
      },
      "source": [
        "## Test Sentences\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZhkSps3rRZj1"
      },
      "source": [
        "sentences = [\n",
        "  \"Go to the bank and deposit your pay check.\",\n",
        "  \"Jim and Janet went down to the river bank to admire the swans.\"\n",
        "]"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "m3nmutoHJvxX",
        "outputId": "07bff6d4-729c-4bc8-bc2d-76582236f459"
      },
      "source": [
        "def get_token_id(sentence, token, show_tokens=False):\n",
        "  tokens = tokenizer.convert_ids_to_tokens(tokenizer(sentence).input_ids)\n",
        "  idx = [idx for idx, tok in enumerate(tokens) if tok == token]\n",
        "  if show_tokens:\n",
        "    print(tokens)\n",
        "  return -1 if len(idx) == 0 else idx[0]\n",
        "\n",
        "\n",
        "get_token_id(sentences[0], \"bank\", show_tokens=True)"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['[CLS]', 'Go', 'to', 'the', 'bank', 'and', 'deposit', 'your', 'pay', 'check', '.', '[SEP]']\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "4"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Jf1CmuQLMjxL",
        "outputId": "cef0c0bb-a063-4cca-d3ae-36829779e03d"
      },
      "source": [
        "def compute_cosine_similarity(x1, x2):\n",
        "  x1 = x1.reshape(1, -1)\n",
        "  x2 = x2.reshape(1, -1)\n",
        "  return torch.nn.CosineSimilarity()(x1, x2).item()\n",
        "\n",
        "\n",
        "x1 = torch.randn(768)\n",
        "x2 = torch.randn(768)\n",
        "compute_cosine_similarity(x1, x2)"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "-0.011603300459682941"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fJIp70vWRDXs"
      },
      "source": [
        "### Cosine Similarity Distributions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rAlLpwdlobNd"
      },
      "source": [
        "def compute_cosine_similarities_for_token(sentence, token):\n",
        "  inputs = tokenizer(sentence, return_tensors=\"pt\")\n",
        "  outputs = model(**inputs, output_hidden_states=True)\n",
        "  # size of embeds_*: (sentence_id=1, num_tokens, 768)\n",
        "  embeds_in, embeds_out = outputs.hidden_states[0], outputs.hidden_states[-1]\n",
        "  token_id = get_token_id(sentence, token)\n",
        "  # size of tok_embed_*: (768)\n",
        "  tok_embed_in, tok_embed_out = embeds_in[0, token_id, :], embeds_out[0, token_id, :]\n",
        "  cosims_in = [compute_cosine_similarity(tok_embed_in, E[i]) for i in range(E.size(0))]\n",
        "  cosims_out = [compute_cosine_similarity(tok_embed_out, E[i]) for i in range(E.size(0))]\n",
        "  return cosims_in, cosims_out\n",
        "\n",
        "\n",
        "cosims_in_0, cosims_out_0 = compute_cosine_similarities_for_token(sentences[0], \"bank\")"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 356
        },
        "id": "zj0XfMpbLgDN",
        "outputId": "f34603f4-70bc-4d9c-d0fb-7800c09ecd48"
      },
      "source": [
        "def plot_cosine_similarity_distribution(cosims_in, cosims_out, sentence):\n",
        "  plt.figure(figsize=(10, 5))\n",
        "\n",
        "  plt.subplot(1, 2, 1)\n",
        "  plt.title(\"input embeddings\")\n",
        "  plt.hist(cosims_in, bins=50)\n",
        "  \n",
        "  plt.subplot(1, 2, 2)\n",
        "  plt.title(\"output embeddings\")\n",
        "  plt.hist(cosims_out, bins=50)\n",
        "\n",
        "  plt.suptitle(sentence)\n",
        "  \n",
        "  _ = plt.show()\n",
        "\n",
        "\n",
        "plot_cosine_similarity_distribution(cosims_in_0, cosims_out_0, sentences[0])"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlwAAAFTCAYAAADsotL4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3debglVXnv8e8vjA7IIB1k0ibSajCJaDpArt7EiAKCCjdXEa/G1pB0TDDDjYmCGVCRBJMbEROHEEHAqEBQQwdQ0oLoNYahURyAEFqE2yBDSzegElH0vX/UOrg5nGF396kz7P5+nmc/p2rVqqq39t5n7XevWrsqVYUkSZL68xNzHYAkSdKoM+GSJEnqmQmXJElSz0y4JEmSembCJUmS1DMTLkmSpJ6ZcEmzJMnNSZ4/Q9uqJHvPxLbGbfe5SW6d6e0Oue+hjynJW5L8Y98xTbH/Nyf5wFztf6Hp4/061+8BaUOZcGlkJDkqyRVJvpvkrjb9O0myEdvapMY8yRlJ3r6x62t+q6q/qKrfAEiyuCUUW851XJLmLxMujYQkbwBOAf4aeAKwC/A64NnA1nMYmjRrTPqk+cuESwteku2BtwG/U1XnVdW3q/OlqnplVT0wVi/JWUnWJrklyZ8mecT/QJJDgDcDL0/ynSRfbuW7JVmRZF2S1Ul+c5J4lgOvBN7Y1v+XgcX7JvlKknuTnJNk24H1XpTkmiT3JPlCkp+b5tAPTXJTkm8l+euxY0ny5CSXJrm7Lftwkh0G9nNzkj+aLI5xx/J7Sa5LsscEyzZpP0n+OMntSb6Z5NenOtAkeyX5bJJvJ1kJ7Dxu+QHtObsnyZeTPHdg2WVJ/jLJlUnuS3J+kp0Glr8kybVt3cuS/PTAsjclua3t94YkB7bywR7Qz7W/97TX+xfHxfaEJPcnefxA2bPa+3CrJD/R3ou3tJ7Zs9p7esJTvBk4Nd3iOC/JPya5D3jNBM/dGUnen2RlO47PJnnSwPJTkqxpz83VSf77MHFPsJ8t0p1q/Xrbz9VJ9hyo8vwkN7bn+T3Jj3uek/x6kuuTrE9y8bj4nt5iX5fkziRvnmDfWyX5aJKPJfELluanqvLhY0E/gEOAB4Etp6l3FnA+sB2wGPhP4OhJ6r4F+MdxZZ8D3gtsC+wLrAWeN8n6ZwBvH1d2M3AlsBuwE3A98Lq27JnAXcD+wBbAslZ/m0m2X8Bn2nae2I7lN9qyvYEXANsAi1rc7xoyjucCt7bpPwe+CCyaJIZN2c8hwJ3AzwCPAT7SjmnvSfb178A7275+Cfj22OsD7A7cDRxK9yXyBW1+UVt+GXDbwL4+NrDuU4DvtnW2At4IrKbrFX0qsAbYrdVdDDx5/PujlRdTvP+Ai4DfHpg/GfjbNv3rbZ8/BTwW+DjwofGvx7jn9fkDcfwAOKId+6MmeS9+uz1v29D1BH9+YPmrgMcDWwJvAO4Atp0u7gn288fAV9vzFuAZwOMH3q8XADvQvV/XAoe0ZYe34//pFsOfAl9oy7YDbm9xbdvm9x98DYBHARe249xirtsjHz4me8x5AD58bOqjfWDcMa7sC8A9wH+1D5otgO8D+wzU+S3gskm2+dAHapvfE/ghsN1A2V8CZ0yy/hlMnHC9amD+r4D3t+n3ASeMq38D8MuTbL/GPrDa/O8Al0xS9wjgS0PG8Vy65OSdwOeB7TfgddiQ/ZwOnDSw7ClMknC1D+gHgccMlH2EHyc8b6IlKAPLLwaWtenLxu1rn/Ze2AL4M+DcgWU/0Y7/uXQJ5V3A84GtJnt/MFzC9XLg39r0FnRJzX5t/hK63tmxuk+lS6K2ZLiE63PTvC5nAGcPzD+W7r285yT11wPPmC7uCda7ATh8ivfrcwbmzwWObdOfZOCLT3sN7geeBLxi8D01wWuwAvgs8G4gw75XffiYi4enFDUK7gZ2zsD4lar6b1W1Q1v2E3SnoLYCbhlY7xa63pFh7Aasq6pvb+T6Y+4YmL6f7sMPug+XN7TTLfckuYcuydttim2tGRfLbgBJdklydjsVdh9dL8DO49adLA7oeiGWA39ZVfdOtvNN3M9uE8Q/md2A9VX13UnqPwl42bjn7jnArgN1xu9rqxbrboPbqqoftbq7V9Vq4A/oPtjvasc61esxlfOBfZLsRdebdm9VXTlwfOPfl1vSjUMcxprpq/y4TlV9B1jHj98vf9RO593bnrvt+fHrOFXc4+0JfH2KGKZ6758y8Nqto+sh232IbR4A/BxdQl1T1JPmnAmXRsG/Aw/QnZqYzLfoeg2eNFD2RLrejImMb7y/CeyUZLuNXH86a4ATq2qHgcejq+qjU6wzOD7miS1GgL9o+//ZqnocXQ/ghvxScz3wIuCDSZ49Rb1N2c/tPDL+qerumOQxk9RfQ9fDNfjcPaaqThqoM35fP6B7T3yTgfdEG1e0J+11raqPVNVzWp0C3jFBfNO+1lX1PbpenVcBvwZ8aGDxw2Lgxz16d9Kd7nz0QHxb0J2+3aD9M3D8SR5Ld4r3m2281huBI4Ed25eUe2mv4zRxj7cGePIQsUy03m+Ne/0eVVVfaMt+aop1/5Wup/mSJMMmqNKcMOHSgldV9wBvBd6b5KVJtmsDkfelG7NDVf2Q7oPjxLb8ScAf0vXKTOROYHHaQPSqWkN3mvIvk2ybbkD70dOsP9UHxXj/ALwuyf7pPCbJYeMSvPH+OMmObWDy7wPntPLtgO8A9ybZnW5szQapqsvoBv5/PMl+k1TblP2cC7wmyT5JHg0cP0UstwCrgLcm2TrJc4AXD1T5R+DFSQ5uA7e3bYPNBwf6v2pgX28Dzht4TxyW5MA2EPwNdMn7F5I8NcnzkmwDfI/u9PSPJghxbSuf7vU+i25Q+0t4eOLyUeB/p/thwGPpEtlzqupBurF527b3wlZ045u2mWY/Ezk0yXPagPITgMvbe3o7uuRuLbBlkj8HHjdk3ON9ADghyZL2Hv65wQH3U3g/cFySp8NDP255WVt2AbBrkj9Isk37391/cOWq+iu6U8yXJBnfwyrNGyZcGgmt0f1Dum/rd7bH39ON7/lCq/a7dD0GN9GNT/oI3ViiifxT+3t3ki+26VfQjdf5JvAJ4Piq+vQk659GdyrmniT/PET8q4DfBP6OrodpNRP84myc84GrgWvoBg2f1srfCjyLrqfiQrpB2BusqlbSDej+lyTPmqDKRu+nqj4JvAu4lO5YL51mlf9F94OCdXTJ2VkD21pD17v5ZrrEYQ1d8jfYvn2IbizTHXSDr3+vrXsDXe/N39L1eL0YeHFVfZ8usTmpld8B/CRw3ATHcj9wIvBv7fU+YJJj/je6xOyLLYkcc3qL73PAN+iSu99t69xLNz7vA3S9bt8FNubCtB+he97WAT/fjhm6sW6fokvsbmn7ftgpyiniHu+ddAnsvwL30b0fHzVdYFX1Cbqew7PbqemvAS9sy75NdyrzxXSvwY3Ar0ywjROAfwY+nWSnJK9Mcu10+5ZmUzztLWmUJbmMboD7nF8ZPsmlwEdmM5YkZ9ANvP/TTdjGrMctjRovkidJsyDJL9D1CE411nDeWahxS/ONpxQlqWdJzgQ+DfzBuF+6zmsLNW5pPvKUoiRJUs/s4ZIkSeqZCZckSVLPTLgkSZJ6ZsIlSZLUMxMuSZKknplwSZIk9cyEawQkuTbJc+c6jpmW5OYkz5+hbZ2R5O1TLK8ke7fp9yf5s5nYryQBtPt7bsxtmSbb3kNt1gTLXpPk8wPz30myIfd2VQ+80vwIqKqnz8Z+ZuIWIQtBVb1urmOQ9EhJClhSVavn4/bmq6p67FzHIHu4JEmSemfCNQIGT70leUuSc5OcleTb7XTj0nF1j0tyXZL1ST6YZNu27GHd0K2skuydZDnwSuCNrXv6XyaJ5WlJViZZl+SGJEcOLDsjyXuTfLJt49+SPCHJu1os/5HkmeM2+QsTxdq296Ik1yS5J8kXkvzcwLJnJvliew7OAbYd3GiSP05ye5JvJvn1ccseOv04dhogyRuS3NXWee1A3ccn+Zck9yW5Ksnbx57DdE5u692X5KtJfmaq11IadUl+Osll7f/22iQvGVh2WZLfGJh/zcD/0+da8Zdb+/Hygf/PNyf5VmvfXrmx25sk3l9Pcn1rgy5O8qSBZZXkd5Lc2NqaE5I8ubVH97W2eOtx25ss1m2S/J8k/y/JnemGNjxqYPlUbdbjk6xo+7wSePK45YNDJs5I8p4kF7aYr0jy5IG6B7W2+97WXn927DlM91nw2bbsW61t1ZBMuEbTS4CzgR2AFcDfjVv+SuBgun/KpwDTniKsqlOBDwN/VVWPraoXj6+T5DHASuAjwE8CRwHvTbLPQLUj2/52Bh4A/h34Yps/D3jnMLG2xOx04LeAxwN/D6xojdbWwD8DHwJ2Av4J+J8DcR4C/BHwAmAJMN04sScA2wO7A0cD70myY1v2HuC7rc6y9hhzEPBLLe7t27HfPc2+pJGVZCvgX4B/pWsjfhf4cJKnTrduVf1Sm3xGa4PGPuyfQNd+7E73/3fqJm5vMN7DgTcDvwosAv4v8NFx1Q4Gfh44AHgjcCrwKmBP4GeAVwzUnSrWk+jain2BvVudP29xTNdmvQf4HrAr8OvtMZWjgLcCOwKrgRPbfsba4ePo2tUbgP82sN4JdK/djsAewN9Osx8NMOEaTZ+vqouq6od0Scczxi3/u6paU1Xr6P7RXvGILWycFwE3V9UHq+rBqvoS8DHgZQN1PlFVV1fV94BPAN+rqrNarOcA43u4Jot1OfD3VXVFVf2wqs6kS+AOaI+tgHdV1Q+q6jzgqoFtHgl8sKq+VlXfBd4yzXH9AHhb29ZFwHeApybZgi6RO76q7q+q64Azx623HfA0uvuWXl9Vt0+zL2mUHQA8Fjipqr5fVZcCF7DpbdCfVdUDVfVZ4EK6//GZ8DrgL9v/7oPAXwD7DvZy0X0Jva+qrgW+BvxrVd1UVfcCn+SRbdojYk0Sujbtf1fVunaj8L+gS4xgijZroB3686r6blV9jYe3QxP5RFVd2Y7pw3RJHsChwLVV9fG27N3AHQPr/QB4ErBbVX2vqh52RkRTM+EaTYP/IPcD2yYZ/IHEmoHpW4DdZmi/TwL2b6cK7klyD10P1RMG6tw5MP1fE8yPH9w5WaxPAt4wbl97tuW7AbfVw+/MfsvA9G4TbHcqd7fGZ8z9Lc5FdD88GdzWQ9Ptw+Tv6L593pXk1CSPm2Zf0ijbDVhTVT8aKLuFrjdnY61vScjg9mayTTtloI1ZB4SHx7shbdpksS4CHg1cPbCvT7VymLrNmqgdmq5NG/8ZMRbjw/bT2tDBX1a+ke74r2yng6frSdMAE67N054D008Evtmmv0v3Tw9AksFECaCY2hrgs1W1w8DjsVX12z3EugY4cdy+Hl1VHwVuB3Zv3xoH1x1z+wTb3RhrgQfputYnipeqendV/TywD93pgj/eyH1Jo+CbwJ5JBj97ngjc1qYf1gbx8C9rk9mxDWcY3N6EbdqQ2xu0Bvitce3Mo6rqCxu4neli/RZdcvb0gf1sP/DrwqnarLF2aCbatNsZaM9aG/rQfFXdUVW/WVW70Q3neG8muTSFHsmEa/N0TJI9kuwE/AndqTyALwNPT7JvusHpbxm33p3AVNdyuQB4SpJfS7JVe/xCkp/uIdZ/AF6XZP90HpPksCTb0Y0LexD4vRbDrwL7DWzzXOA1SfZJ8mjg+I0JrJ0G/TjwliSPTvI04NVjy9ux79/GrXyXbozFjybemrRZuIKuR+WN7X/zucCL6cacAlwD/Gr7f9qbbszkoMnaoLcm2TrJf6cb2vBPm7i9Me8HjkvydIAk2yd52RT1h/GIWFuP3z8AJyf5ybav3ZMc3NaZtM2aoB3ah4ePJd0QFwI/m+SIdlbkGAaS1CQvSzKWgK2n+xJumzYkE67N00foBj7eBHwdeDtAVf0n8Dbg08CNwPjz86cB+7Qu738ev9E27uAgunEH36Trtn4HsE0Psa4CfpPulN16uoGfr2nLvk83yPU1dKcAXk7XII3F+UngXcClbb1LNyG+19MNiL+DbrzcR+nGkgE8jq4RXU/XxX838NebsC9pQWv/my8GXkjXq/Ne4NVV9R+tysnA9+kSoTPpxhcNegtwZmuDxsZp3UH3P/bNVv91m7i9wXg/QdeGnZ3kProxWi/c8CN/yFSxvomuPbq87evTwFNbHNO1Wa+nOy14B3AG8MGNCa6qvkU35vav6NqrfYBV/LhN+wXgiiTfoftB1u9X1U0bs6/NUR4+zEWjLsnNwG9U1afnOpZRlOQdwBOqamO/YUoaUush+8eq2mO6utpw7dTvrcArq+ozcx3PQmcPl7QJ0l137Ofaac396E5ZfGKu45KkjZHk4CQ7JNmG7pIYAS6f47BGgrf2kTbNdnSnEXejO23xN8D5cxqRJG28X6QbyrE1cB1wRFX919yGNBo8pShJktQzTylKkiT1zIRLkiSpZ/N6DNfOO+9cixcvnuswJM2iq6+++ltVtWj6mvOfbZi0eZmq/ZrXCdfixYtZtWrVXIchaRYlme62JAuGbZi0eZmq/fKUoiRJUs9MuCRJknpmwiVJktQzEy5JkqSemXBJkiT1zIRLkiSpZyZckiRJPTPhkiRJ6pkJlyRJUs9MuCRJknpmwiVJktSzeX0vxflu8bEXPmz+5pMOm6NIJGl22O5JG8ceLkmSpJ6ZcEmSJPXMhEvSyEuyRZIvJbmgze+V5Iokq5Ock2TrVr5Nm1/dli8e2MZxrfyGJAfPzZFIWqgcwyVpc/D7wPXA49r8O4CTq+rsJO8Hjgbe1/6ur6q9kxzV6r08yT7AUcDTgd2ATyd5SlX9cLYPZL4ZP6YLHNclTcQeLkkjLckewGHAB9p8gOcB57UqZwJHtOnD2zxt+YGt/uHA2VX1QFV9A1gN7Dc7RyBpFJhwSRp17wLeCPyozT8euKeqHmzztwK7t+ndgTUAbfm9rf5D5ROsI0nTGirhSrJDkvOS/EeS65P8YpKdkqxMcmP7u2OrmyTvbmMdvpLkWQPbWdbq35hkWV8HJUkASV4E3FVVV8/iPpcnWZVk1dq1a2drt5LmuWF7uE4BPlVVTwOeQTcW4ljgkqpaAlzS5gFeCCxpj+V04yJIshNwPLA/XVf88WNJmiT15NnAS5LcDJxNdyrxFGCHJGNjWPcAbmvTtwF7ArTl2wN3D5ZPsM7DVNWpVbW0qpYuWrRoZo9G0oI1bcKVZHvgl4DTAKrq+1V1Dw8f6zB+DMRZ1bmcrmHbFTgYWFlV66pqPbASOGRGj0aSBlTVcVW1R1Utphv0fmlVvRL4DPDSVm0ZcH6bXtHmacsvrapq5Ue1XzHuRfeF8spZOgxJI2CYHq69gLXAB9vPqj+Q5DHALlV1e6tzB7BLm55srINjICTNF28C/jDJaroxWqe18tOAx7fyP6T13FfVtcC5wHXAp4Bj/IWipA0xzGUhtgSeBfxuVV2R5BR+fPoQgKqqJDUTASVZTncqkic+8YkzsUlJoqouAy5r0zcxwa8Mq+p7wMsmWf9E4MT+IpQ0yobp4boVuLWqrmjz59ElYHe2U4W0v3e15ZONdRhqDITjHyRJ0qiZNuGqqjuANUme2ooOpOtWHxzrMH4MxKvbrxUPAO5tpx4vBg5KsmMbLH9QK5MkSRppw15p/neBD7fbX9wEvJYuWTs3ydHALcCRre5FwKF0Fwa8v9WlqtYlOQG4qtV7W1Wtm5GjkCRJmseGSriq6hpg6QSLDpygbgHHTLKd04HTNyRASZKkhc4rzUuSJPXMhEuSJKlnJlySJEk9M+GSJEnq2bC/UpQkaSiLj73wEWU3n3TYHEQizR/2cEmSJPXMhEuSJKlnJlySJEk9M+GSJEnqmQmXJElSz0y4JEmSeuZlIYY00c+cJUmShmEPlyRJUs9MuCRJknpmwiVJktQzEy5JkqSemXBJkiT1zIRLkiSpZyZckiRJPTPhkjSykmyb5MokX05ybZK3tvIzknwjyTXtsW8rT5J3J1md5CtJnjWwrWVJbmyPZXN1TJIWJi98KmmUPQA8r6q+k2Qr4PNJPtmW/XFVnTeu/guBJe2xP/A+YP8kOwHHA0uBAq5OsqKq1s/KUUha8OzhkjSyqvOdNrtVe9QUqxwOnNXWuxzYIcmuwMHAyqpa15KslcAhfcYuabSYcEkaaUm2SHINcBdd0nRFW3RiO214cpJtWtnuwJqB1W9tZZOVT7S/5UlWJVm1du3aGT0WSQuXCZekkVZVP6yqfYE9gP2S/AxwHPA04BeAnYA3zeD+Tq2qpVW1dNGiRTO1WUkLnAmXpM1CVd0DfAY4pKpub6cNHwA+COzXqt0G7Dmw2h6tbLJySRqKCZekkZVkUZId2vSjgBcA/9HGZZEkwBHA19oqK4BXt18rHgDcW1W3AxcDByXZMcmOwEGtTJKG4q8UJY2yXYEzk2xB9wXz3Kq6IMmlSRYBAa4BXtfqXwQcCqwG7gdeC1BV65KcAFzV6r2tqtbN4nFIWuBMuCSNrKr6CvDMCcqfN0n9Ao6ZZNnpwOkzGuA8t/jYC+c6BGlkeEpRkiSpZyZckiRJPTPhkiRJ6pkJlyRJUs8cND+DJhpgevNJh81BJJIkaT4Zqocryc1JvprkmiSrWtlOSVYmubH93bGVJ8m7k6xut8141sB2lrX6NyZZ1s8hSZIkzS8bckrxV6pq36pa2uaPBS6pqiXAJW0e4IXAkvZYDrwPugQNOB7Yn+6qzsePJWmSJEmjbFPGcB0OnNmmz6S7WvNY+VntthmXAzu0qzofTHfj2HVVtR5YCRyyCfuXJElaEIZNuAr41yRXJ1neynZpt7wAuAPYpU3vDqwZWPfWVjZZ+cMkWZ5kVZJVa9euHTI8SZKk+WvYQfPPqarbkvwksDLJfwwurKpKUjMRUFWdCpwKsHTp0hnZpiRJ0lwaqoerqm5rf+8CPkE3BuvOgRvA7grc1arfBuw5sPoerWyyckmSpJE2bcKV5DFJthubBg4CvgasAMZ+abgMOL9NrwBe3X6teABwbzv1eDFwUJId22D5g1qZJEnSSBvmlOIuwCeSjNX/SFV9KslVwLlJjgZuAY5s9S8CDgVWA/cDrwWoqnVJTgCuavXeVlXrZuxIJEmS5qlpE66qugl4xgTldwMHTlBewDGTbOt04PQND1OSJGnh8tY+kiRJPTPhkiRJ6pkJlyRJUs9MuCRJknpmwiVJktQzEy5JkqSemXBJkiT1zIRLkiSpZyZckkZWkm2TXJnky0muTfLWVr5XkiuSrE5yTpKtW/k2bX51W754YFvHtfIbkhw8N0ckaaEa5tY+krRQPQA8r6q+k2Qr4PNJPgn8IXByVZ2d5P3A0cD72t/1VbV3kqOAdwAvT7IPcBTwdGA34NNJnlJVP5yLg1qIFh974cPmbz7psDmKRJob9nBJGlnV+U6b3ao9CngecF4rPxM4ok0f3uZpyw9MdyPZw4Gzq+qBqvoG3b1i95uFQ5A0Iky4JI20JFskuQa4C1gJfB24p6oebFVuBXZv07sDawDa8nuBxw+WT7COJE3LhEvSSKuqH1bVvsAedL1ST+tzf0mWJ1mVZNXatWv73JWkBcSES9JmoaruAT4D/CKwQ5KxMax7ALe16duAPQHa8u2BuwfLJ1hn/H5OraqlVbV00aJFM34ckhYmEy5JIyvJoiQ7tOlHAS8ArqdLvF7aqi0Dzm/TK9o8bfmlVVWt/Kj2K8a9gCXAlbNzFJJGgb9SlDTKdgXOTLIF3RfMc6vqgiTXAWcneTvwJeC0Vv804ENJVgPr6H6ZSFVdm+Rc4DrgQeAYf6EoaUOYcEkaWVX1FeCZE5TfxAS/Mqyq7wEvm2RbJwInznSMkjYPnlKUJEnqmQmXJElSz0y4JEmSembCJUmS1DMTLkmSpJ6ZcEmSJPXMhEuSJKlnJlySJEk9M+GSJEnqmQmXJElSz0y4JEmSembCJUmS1DMTLkmSpJ6ZcEmSJPXMhEuSJKlnQydcSbZI8qUkF7T5vZJckWR1knOSbN3Kt2nzq9vyxQPbOK6V35Dk4Jk+GEmSpPloQ3q4fh+4fmD+HcDJVbU3sB44upUfDaxv5Se3eiTZBzgKeDpwCPDeJFtsWviSJEnz31AJV5I9gMOAD7T5AM8DzmtVzgSOaNOHt3na8gNb/cOBs6vqgar6BrAa2G8mDkKSJGk+G7aH613AG4EftfnHA/dU1YNt/lZg9za9O7AGoC2/t9V/qHyCdSRJkkbWtAlXkhcBd1XV1bMQD0mWJ1mVZNXatWtnY5eSJEm9GqaH69nAS5LcDJxNdyrxFGCHJFu2OnsAt7Xp24A9Adry7YG7B8snWOchVXVqVS2tqqWLFi3a4AOSJEmab6ZNuKrquKrao6oW0w16v7SqXgl8Bnhpq7YMOL9Nr2jztOWXVlW18qParxj3ApYAV87YkUiSJM1TW05fZVJvAs5O8nbgS8Bprfw04ENJVgPr6JI0quraJOcC1wEPAsdU1Q83Yf+SJEkLwgZd+LSqLquqF7Xpm6pqv6rau6peVlUPtPLvtfm92/KbBtY/saqeXFVPrapPzuyhSNLDJdkzyWeSXJfk2iS/38rfkuS2JNe0x6ED60x4vcAkh7Sy1UmOnYvjkbRwbUoPlyTNdw8Cb6iqLybZDrg6ycq27OSq+j+DlcddL3A34NNJntIWvwd4Ad0vrK9KsqKqrpuVo5C04JlwSRpZVXU7cHub/naS65n6cjQPXS8Q+EYbGjF2vcDVYz32Sc5udU24JA3FeylK2iy024w9E7iiFb0+yVeSnJ5kx1Y22fUCh76OoJe2kTQREy5JIy/JY4GPAX9QVfcB7wOeDOxL1wP2NzO1Ly9tI2kinlKUNNKSbEWXbH24qj4OUFV3Diz/B+CCNjvV9QKnvY6gJE3GHi5JI6vdx/U04PqqeudA+a4D1f4H8LU2Pdn1Aq8CliTZK8nWdAPrV8zGMUgaDfZwSRplzwZ+Dfhqkmta2ZuBVyTZFyjgZuC3YOrrBSZ5PXAxsAVwelVdO5sHImlhM+GSNLKq6vNAJlh00RTrnAicOEH5RVOtJ0lT8ZSiJElSz0y4JEmSembCJUmS1DMTLkmSpJ6ZcDIV8ukAABN0SURBVEmSJPXMhEuSJKlnXhZCksTiYy+c6xCkkWYPlyRJUs9MuCRJknpmwiVJktQzx3BNwLEMktSvidrZm086bA4ikWaHPVySJEk9M+GSJEnqmQmXJElSz0y4JEmSembCJUmS1DMTLkmSpJ6ZcEmSJPXMhEuSJKlnJlySJEk9M+GSJEnqmQmXJElSz7yXYs/G3y/Me4VJsyfJnsBZwC5AAadW1SlJdgLOARYDNwNHVtX6JAFOAQ4F7gdeU1VfbNtaBvxp2/Tbq+rM2TwWSQubPVySRtmDwBuqah/gAOCYJPsAxwKXVNUS4JI2D/BCYEl7LAfeB9AStOOB/YH9gOOT7DibByJpYZs24UqybZIrk3w5ybVJ3trK90pyRZLVSc5JsnUr36bNr27LFw9s67hWfkOSg/s6KEkCqKrbx3qoqurbwPXA7sDhwFgP1ZnAEW36cOCs6lwO7JBkV+BgYGVVrauq9cBK4JBZPBRJC9wwPVwPAM+rqmcA+wKHJDkAeAdwclXtDawHjm71jwbWt/KTWz3at8qjgKfTNVTvTbLFTB6MJE2mffl7JnAFsEtV3d4W3UF3yhG6ZGzNwGq3trLJyiVpKNMmXO2b3nfa7FbtUcDzgPNa+fhviGPfHM8DDmzjIg4Hzq6qB6rqG8Bquq55SepVkscCHwP+oKruG1xWVUXXps3UvpYnWZVk1dq1a2dqs5IWuKHGcCXZIsk1wF10XelfB+6pqgdblcFvew99E2zL7wUej98QJc2BJFvRJVsfrqqPt+I726lC2t+7WvltwJ4Dq+/RyiYrf4SqOrWqllbV0kWLFs3cgUha0IZKuKrqh1W1L10jsx/wtL4C8tuhpJnSetdPA66vqncOLFoBLGvTy4DzB8pfnc4BwL3t1OPFwEFJdmyD5Q9qZZI0lA26LERV3ZPkM8Av0g0m3bL1Yg1+2xv7Jnhrki2B7YG7GfIbYlWdCpwKsHTp0hnr5pe0WXo28GvAV1svPcCbgZOAc5McDdwCHNmWXUR3SYjVdJeFeC1AVa1LcgJwVav3tqpaNzuHIGkUTJtwJVkE/KAlW48CXkA3EP4zwEuBs3nkN8RlwL+35ZdWVSVZAXwkyTuB3eh+dn3lDB+PJD2kqj4PZJLFB05Qv4BjJtnW6cDpMxedpM3JMD1cuwJntl8U/gRwblVdkOQ64Owkbwe+RNdtT/v7oSSrgXV0v0ykqq5Nci5wHd21cY6pqh/O7OFIkiTNP9MmXFX1FbqfUo8vv4kJfmVYVd8DXjbJtk4ETtzwMCVJkhYurzQvSZLUMxMuSZKknplwSZIk9cyES5IkqWcmXJIkST0z4ZIkSeqZCZckSVLPTLgkSZJ6ZsIlSZLUMxMuSZKknplwSZIk9cyES5IkqWcmXJIkST0z4ZIkSeqZCZckSVLPTLgkSZJ6ZsIlSZLUMxMuSZKknplwSZIk9cyES5IkqWcmXJIkST0z4ZI00pKcnuSuJF8bKHtLktuSXNMehw4sOy7J6iQ3JDl4oPyQVrY6ybGzfRySFjYTLkmj7gzgkAnKT66qfdvjIoAk+wBHAU9v67w3yRZJtgDeA7wQ2Ad4RasrSUPZcq4DkKQ+VdXnkiwesvrhwNlV9QDwjSSrgf3astVVdRNAkrNb3etmOFxJI8qES9Lm6vVJXg2sAt5QVeuB3YHLB+rc2soA1owr33+ijSZZDiwHeOITnzjTMc+YxcdeONchSJsVTylK2hy9D3gysC9wO/A3M7Xhqjq1qpZW1dJFixbN1GYlLXD2cEna7FTVnWPTSf4BuKDN3gbsOVB1j1bGFOWSNC17uCRtdpLsOjD7P4CxXzCuAI5Ksk2SvYAlwJXAVcCSJHsl2ZpuYP2K2YxZ0sJmD5ekkZbko8BzgZ2T3AocDzw3yb5AATcDvwVQVdcmOZduMPyDwDFV9cO2ndcDFwNbAKdX1bWzfCiSFjATLkkjrapeMUHxaVPUPxE4cYLyi4CLZjA0SZsRTylKkiT1zIRLkiSpZ9OeUkyyJ3AWsAvdeIdTq+qUJDsB5wCL6cZAHFlV65MEOAU4FLgfeE1VfbFtaxnwp23Tb6+qM2f2cCRJC9X4a4PdfNJhcxSJNPOG6eF6kO6igPsABwDHtFtaHAtcUlVLgEvaPHS3vljSHsvprndDS9COp7tY4H7A8Ul2nMFjkSRJmpemTbiq6vaxHqqq+jZwPd2Vlw8HxnqozgSOaNOHA2dV53Jgh/YT7IOBlVW1rl3ReSUT399MkiRppGzQGK52P7JnAlcAu1TV7W3RHXSnHKFLxsbfAmP3KcolSZJG2tAJV5LHAh8D/qCq7htcVlVFN75rkyVZnmRVklVr166diU1KkiTNqaESriRb0SVbH66qj7fiO8eu1tz+3tXKJ7s1xlS3zHiI9yGTJEmjZtqEq/3q8DTg+qp658CiFcCyNr0MOH+g/NXpHADc2049XgwclGTHNlj+oFYmSZI00oa50vyzgV8Dvprkmlb2ZuAk4NwkRwO3AEe2ZRfRXRJiNd1lIV4LUFXrkpxAd08ygLdV1boZOQpJkqR5bNqEq6o+D2SSxQdOUL+AYybZ1unA6RsSoCRJ0kLnleYlSZJ6ZsIlSZLUMxMuSZKknplwSZIk9cyES5IkqWfDXBZCM2jxsRc+ouzmkw6bg0gkSdJssYdLkiSpZyZckiRJPTPhkiRJ6pkJlyRJUs9MuCRJknpmwiVppCU5PcldSb42ULZTkpVJbmx/d2zlSfLuJKuTfCXJswbWWdbq35hk2Vwci6SFy4RL0qg7AzhkXNmxwCVVtQS4pM0DvBBY0h7LgfdBl6ABxwP7A/sBx48laZI0DBMuSSOtqj4HrBtXfDhwZps+EzhioPys6lwO7JBkV+BgYGVVrauq9cBKHpnESdKkTLgkbY52qarb2/QdwC5tendgzUC9W1vZZOWSNBQTLkmbtaoqoGZqe0mWJ1mVZNXatWtnarOSFjgTLkmbozvbqULa37ta+W3AngP19mhlk5U/QlWdWlVLq2rpokWLZjxwSQuT91Jk4vsbShppK4BlwEnt7/kD5a9PcjbdAPl7q+r2JBcDfzEwUP4g4LhZjlnSAmbCJWmkJfko8Fxg5yS30v3a8CTg3CRHA7cAR7bqFwGHAquB+4HXAlTVuiQnAFe1em+rqvED8SVpUiZckkZaVb1ikkUHTlC3gGMm2c7pwOkzGJqkzYhjuCRJknpmwiVJktQzEy5JkqSemXBJkiT1zEHzkjTivPSNNPfs4ZIkSeqZPVySpHlpop65m086bA4ikTadPVySJEk9M+GSJEnqmQmXJElSz0y4JEmSembCJUmS1DMTLkmSpJ5Nm3AlOT3JXUm+NlC2U5KVSW5sf3ds5Uny7iSrk3wlybMG1lnW6t+YZFk/hyNJkjT/DNPDdQZwyLiyY4FLqmoJcEmbB3ghsKQ9lgPvgy5BA44H9gf2A44fS9IkSZJG3bQJV1V9Dlg3rvhw4Mw2fSZwxED5WdW5HNghya7AwcDKqlpXVeuBlTwyiZMkSRpJGzuGa5equr1N3wHs0qZ3B9YM1Lu1lU1WLkmSNPI2edB8VRVQMxALAEmWJ1mVZNXatWtnarOSJElzZmMTrjvbqULa37ta+W3AngP19mhlk5U/QlWdWlVLq2rpokWLNjI8SZKk+WNjE64VwNgvDZcB5w+Uv7r9WvEA4N526vFi4KAkO7bB8ge1MkmSpJG35XQVknwUeC6wc5Jb6X5teBJwbpKjgVuAI1v1i4BDgdXA/cBrAapqXZITgKtavbdV1fiB+JIkSSNp2oSrql4xyaIDJ6hbwDGTbOd04PQNik6SJGkEeKV5SZKknplwSdpsJbk5yVeTXJNkVSvb4DtpSNJ0pj2lqP4tPvbCR5TdfNJhcxCJtFn6lar61sD82J00TkpybJt/Ew+/k8b+dHfS2H+2g5W0MNnDJUkPt6F30pCkaZlwSdqcFfCvSa5OsryVbeidNB7GizdLmoinFCVtzp5TVbcl+UlgZZL/GFxYVZVkg+6kUVWnAqcCLF26dMbuwiFpYbOHS9Jmq6pua3/vAj4B7MeG30lDkqZlwiVps5TkMUm2G5umuwPG19jwO2lI0rQ8pShpc7UL8Ikk0LWFH6mqTyW5ig24k4YkDcOES9JmqapuAp4xQfndbOCdNCRpOiZckqQFw+sWaqFyDJckSVLP7OGSpBEzUS+QpLllD5ckSVLPTLgkSZJ6ZsIlSZLUMxMuSZKknplwSZIk9cyES5IkqWeb3WUhFsrPpcfH6YX9JElauOzhkiRJ6pkJlyRJUs82u1OKkqTR4hAMLQT2cEmSJPXMhEuSJKlnJlySJEk9cwzXAjHR5SwcpyBpoVzqRtrc2cMlSZLUMxMuSZKknnlKUZI0UhyCofnIHi5JkqSe2cO1gHmxP0mSFoZZT7iSHAKcAmwBfKCqTprtGCRpY8x1++UvEjeeX1A112Y14UqyBfAe4AXArcBVSVZU1XV97M/GSdJMme32S9Jome0erv2A1VV1E0CSs4HDARusGeBAUalXtl8jxPZSs222E67dgTUD87cC+8/Uxu3ReqRhnhMbGWkovbZfE7FNm122l+rTvBs0n2Q5sLzNfifJDW16Z+BbcxPVBllwceYdcxzJ9BbcczrPzfc4nzTXAWyKKdqwPsz31xLmf4wbFN8ctJcj9fzNgdmOb9L2a7YTrtuAPQfm92hlD6mqU4FTx6+YZFVVLe03vE1nnDNvocRqnCNv2vYLJm/D+rAQXsv5HqPxbRrjG95sX4frKmBJkr2SbA0cBayY5RgkaWPYfknaaLPaw1VVDyZ5PXAx3c+qT6+qa2czBknaGLZfkjbFrI/hqqqLgIs2YtVZ6aKfAcY58xZKrMY54jah/erLQngt53uMxrdpjG9Iqaq5jkGSJGmkeS9FSZKkns3bhCvJTklWJrmx/d1xgjr7Jvn3JNcm+UqSl89ifIckuSHJ6iTHTrB8myTntOVXJFk8W7GNi2O6OP8wyXXt+bskyZz9JH+6WAfq/c8klWROfnkyTJxJjmzP67VJPjLbMbYYpnvtn5jkM0m+1F7/Q+ciTg1vmHax1ftUknuSXDBLcc3r9nCI+H4pyReTPJjkpbMZ2wbEOKdt9RDxvS7JV5Nck+TzSfaZT/EN1Ju7z4+qmpcP4K+AY9v0scA7JqjzFGBJm94NuB3YYRZi2wL4OvBTwNbAl4F9xtX5HeD9bfoo4Jw5eA6HifNXgEe36d+eiziHjbXV2w74HHA5sHQ+xgksAb4E7Njmf3Kexnkq8Ntteh/g5rl47X1s0Os6bbvYlh0IvBi4YBZimtft4ZDxLQZ+DjgLeOkcvK7zuq0eMr7HDUy/BPjUfIqv1ZvTz49528NFd8uMM9v0mcAR4ytU1X9W1Y1t+pvAXcCiWYjtoVt8VNX3gbFbfAwajP884MAkmYXYBk0bZ1V9pqrub7OX011baC4M85wCnAC8A/jebAY3YJg4fxN4T1WtB6iqu2Y5RhguzgIe16a3B745i/Fp40zbLgJU1SXAt2cppvneHg7TDt5cVV8BfjRLMY0339vqYeK7b2D2MXTty7yJr5nTz4/5nHDtUlW3t+k7gF2mqpxkP7rM9ut9B8bEt/jYfbI6VfUgcC/w+FmIbcIYmoniHHQ08MleI5rctLEmeRawZ1XN5f1OhnlOnwI8Jcm/Jbk8ySGzFt2PDRPnW4BXJbmV7pd3vzs7oWkTbFC7OEvme3u4oe3gXJjvbfVQ8SU5JsnX6Xpif2+WYoMF8vkxp7f2SfJp4AkTLPqTwZmqqiSTZstJdgU+BCyrqrn6hrKgJXkVsBT45bmOZSJJfgJ4J/CaOQ5lGFvSnVZ8Lt230M8l+dmqumdOo3qkVwBnVNXfJPlF4ENJfsb/obk1U+2iRtN8bqur6j3Ae5L8L+BPgWVzHBIwfz4/5jThqqrnT7YsyZ1Jdq2q21tCNeFpmSSPAy4E/qSqLu8p1PGGucXHWJ1bk2xJd8rm7tkJ7xExjJnwViRJnk/XmP9yVT0wS7GNN12s2wE/A1zWzkQ8AViR5CVVtWrWohzuOb0VuKKqfgB8I8l/0iVgV81OiMBwcR4NHAJQVf+eZFu6+47NxSlQNTPRLs6y+d4eDtUOzrH53lZv6HN4NvC+XiN6uAXx+TGfTymu4MfZ8TLg/PEV0t1e4xPAWVV13izGNswtPgbjfylwabVRe7No2jiTPBP4e+AlczTWaMyUsVbVvVW1c1UtrqrFdGMYZjvZmjbO5p/perdIsjPdKcabZjNIhovz/9ENribJTwPbAmtnNUptqGnbxTkw39vDhXBLpvneVg8T35KB2cOAG+dLfPPm82O2R+kP+6A7v38J3Yv2aWCnVr4U+ECbfhXwA+Cagce+sxTfocB/0o0Z+5NW9ja6FxG6D69/AlYDVwI/NUfP43Rxfhq4c+D5WzGHr/mUsY6rexlz8CuTIZ/T0HVfXwd8FThqnsa5D/BvdL/ouQY4aK5eex9Dv6bTtott/v/SJc//RdfjenDPcc3r9nCI+H6hPU/fpet5u3YOXtt53VYPEd8pwLUtts8AT59P8Y2rOyefH15pXpIkqWfz+ZSiJEnSSDDhkiRJ6pkJlyRJUs9MuCRJknpmwiVJktQzEy5JkqSemXBJkiT1zIRLkiSpZ/8fak+mS93q45cAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 720x360 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 356
        },
        "id": "REHLLHqtNdh1",
        "outputId": "0e9f6b9b-8c61-4caa-8f01-6dc0cb766315"
      },
      "source": [
        "cosims_in_1, cosims_out_1 = compute_cosine_similarities_for_token(sentences[1], \"bank\")\n",
        "plot_cosine_similarity_distribution(cosims_in_1, cosims_out_1, sentences[1])"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlwAAAFTCAYAAADsotL4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de5wcVZ338c/XcBNBEmCMkASCEC/B1cBGwMddF0VDADWsIsJ6CRiNumF1V1cN6gqCuLDPKsizgEaJBAUCokgEBMNdVC7hKglgBgibhEACuXBHA7/nj3MGKp3udE9mqnu65/t+vfo1VadOVf2qe/r0r0+d6lJEYGZmZmbleUWrAzAzMzPrdE64zMzMzErmhMvMzMysZE64zMzMzErmhMvMzMysZE64zMzMzErmhKvDSfp7Sfe1Oo5qJC2S9J5Wx9GOJI2WFJI2aXUs/UXSsZJ+VuL2fyDpP8rafsW+SvnfHsivu6SzJH17I9f9mqQf93dMhe0fIemGsrZv1ggnXB1K0rWSPhURv4uIN7Q6nt4q+8O32ftpII6QtFur42hUXz/4Je0raUl/x7UhEfHZiDi+mftspVY8xxsrIr4TEZ/qj20N5KTUBjcnXGbW8SQNKWm7kuR2tEROnKxTuKHocJXfcvOpji9LukvS05LOlDRc0m8kPSnpSknDamxrmKRLJK2QtCpPjywsv1bS8ZJ+n7f1W0nbF5Z/XNJDkh6X9PVeHsf3JS2W9ISkWyX9fWHZsZIukHR23u98SeMLy3eU9Isc94OSPp/LJwJfAz4i6SlJd1bZ75GSfl2YXyjp54X5xZLG5ek3SporaaWk+yQdWqh3lqTTJF2aY7xJ0q552fW52p05jo9UiWOIpP+W9JikB4CDKpbvKGlO3ne3pE/n8i0kPdvzOkj6uqS1kl6d54+XdEq9GKvoiXl1jvntkl4h6Rv5NV6eX49tqhzLq4DfADvmdZ+StGNevFlvX8dq8rGcIekySU8D71LhlJekeyS9r1B/k7zdPfP8PpL+IGm1pDsl7Vuoe62kEyT9HngGeF2NMN4maUF+r/xE0hZ5/T69jyqO80NK7+k3N/IcS9pc0imSHs6PUyRtXmPbu0q6Wun9+pikcyQNLSzfQ9JtOcbzgS0Ky/aVtETSV/L/wjJJB0s6UNKf8//p1wr1X+pp1ss9VFMk/S9wdS7/ZH7dVkm6QtLONZ739f43C/v577z+g5IOKJRvo9QWLpO0VNK3VSNJl7SXpHlKbdGjkr6Xy2dJ+lKeHpGPYVrhuVyZ3yMb/forvZ9/ll+T1ZJukTS8xvNgA01E+NGBD+Ba4FPAvsCSQvki4EZgODACWA7cBuxBajCvBo6psc3tgA8BWwJbAz8HflWxz/uB1wOvzPMn5mVjgaeAdwKbA98D1gLvqbGvY4GfFeY/lve/CfAl4BFgi0Ld54ADgSHAfwI35mWvAG4FvglsRvpwfADYv9p+qsTxOmB13s6OwEM9z2detiovexWwGDgyx7gH8BgwNtc9C3gc2CsvPweYXdhPALttII7PAvcCo4BtgWvyOpvk5dcDp+fXcBywAnh3YdmH8vRv82t0QGHZPzYSY0U8o4v7z2WfBLrz87IV8EvgpzXW35fC/2VfX8cq2z8LWAO8I6+7RS77dl7+TeCcQv2DgHvy9Ij8PByY131vnu8q/J//L7B7fp42rbL/RcDdhdfr94V99+V99NLzTvpf6671f1PjOT6O9P5/DdAF/AE4vsb6u+Vj3zzXvR44JS/bjPRe+DdgU+AQ4K+FY9yX9P7+Zl7+adL/5Ln5mHcHngV2qXwfFo7xbNL76pXApHysb8rH/g3gD7343zwix/dp0v/W54CHAeXlFwE/zPt7DXAz8Jka2/8j8PE8vRWwT+H//9d5+p/ya3h+YdnF/fD6fwb4dV53CPC3wKs39nPCj+Y+Wh6AHyW9sBtOuD5amP8FcEZh/l+Kb/46+xgHrKrY5zcK8/8MXJ6nv8m6CcargL/QYMJVZfkq4K2FulcWlo0Fns3TewP/W7Hu0cBPGtlPrrMY2BM4DJiRG+M3kj7w5uQ6HwF+V7HeD8nJK+nD/seFZQcC9xbm6yVcVwOfLcxP4OUP3lHAC8DWheX/CZyVp48HTs11HwG+AJxISkKeBbZrJMaKeEaz/ofaVcA/F+bfQPqQ26TK+uv8X/b1dayy/bOAs6uU9SQEuwFPAlvm+XOAb+bpr1KRKAJXAJML/+fH1fmfWVTxeh0I3N8P76Oe5/3fgQXAyA3EUO05vh84sDC/P7BoQ8dSqHswcHuefieFhCWX/YF1E65ngSF5fusc996F+rcCB1e+DwvH+LpC3d8AUwrzryD1Lu7c4P/mEUB3YX7LXOe1pC+fzwOvLCw/HLimxvNwPfAtYPuK8l15+QvYD0jJUc+Xs1nAF/vh9f9kfp7f0shr5sfAeviU4uD0aGH62SrzW1VbSdKWkn6odMroCVLDM7Si6/2RwvQzhW3tSEpcAIiIp0m9Bg2R9O/5dMIaSauBbYDiaZbK/W6hNPZjZ9JpldU9D9JpxN50w19H+gB5Z56+FviH/Lgu19kZ2LtiPx8lNei1Yqz6PNewzvNH6l0oLlsZEU9WLB9REf+ewJ+AuTn2fUgfQsXXoa8xFuN6iJTk9ea57s/XcXGtBRHRDdwDvF/SlsAHSL0v5H19uGJffwfs0Mi2a9R5iPT89PV91OPLwGkR0dtB8dVeox2rVVQaajA7n2J7AvgZL7/ndgSWRs4CCtsqejwiXsjTz+a/DbU1WfH52xn4fuH1WAmIl//HG/HScxoRz+TJrfK2NwWWFbb/Q1JPVzVTSL1P9+ZTeu/L27wfeJqUQP09cAnwsKQ3UGgr+vj6/5SU/M/Op4T/S9KmvXgOrIU8GNF640ukXou9I+IRpbFLt5MavnqWkU4HAKnRIXWt16U0XusrwH7A/Ih4UdKqBve7GHgwIsbUWB41youuA94P7AJ8h3SK8aPA24H/Keznuoh4bwPb2xjLSD1ZPXYqTD8MbCtp60LStROwNE//gfS6/WOOcYGknUi9Ltexcao9bw+TPryKMa5l3Q/ZDa2/IfVex2rq7eM8Uk/GK4AFOQnr2ddPI+LTfdg2rP96PZyn+/I+6jEBuFzSIxHxi17E2PMaza8SV6Xv5G38TUSslHQwL/+/LwNGSFIh6dqJ1IPWX4rxLwZOiIhzerleIxaTeri2j4i1dTcesRA4XOliiQ8CF0raLn+JvI50enWziFgq6TpgMjAMuCNvYqNf/4j4K6l37VuSRgOXAfcBZ/bieK1F3MNlvbE16VvpaknbAsf0Yt0LgfdJ+jtJm5HGkjT6/7c16YN7BbCJpG8Cr25w3ZuBJyV9VdIrlQafv1nS2/LyR4HR2vCVZtcB7yKdclgC/A6YSEoYb891LgFer3RhwKb58TZJb6q+yfU8Su3B1wAXAJ+XNFLpoobpPQsiYjEpqfrPPKj2LaRv4T/Ly58hnb6ZxssJ1h9I48I2NuFaAbxYEfN5wL9J2kXSVqQP7PNrfIg9CmynKoPqa6j3Om6M2aTE5XO83LsF6Xl7v6T98362UBoEPrLqVmqbll+vbYGvA+fn8r68j3rMJ/0PnibpAzXqVHuOzwO+IakrD8T+Jvn/pIqtSeMu10gaQepV6/FH0nvy8/l//YOksX9l+QFwtKTd4aVB7h+uUbfa/2ZNEbGMNLbxu5JerTSwfVdJ/1CtvqSPSeqKiBdJX77I+4P0fjqKlwfuX5vnbyj09m306y/pXZL+JveGPUE6Zf9indVsgHDC1dl6+02vnlNIgzgfIw28vbzhQCLmkz7wzyV9O14F1Dsd0hP/FXlffyadtniOxk7pkBu595G6+R/Msf+YdEoS0oBVgMcl3VZjG38mffD8Ls8/QRqw/fueRjT3LE0gjfN6mHRK4CTSgONGHAvMyqc0Dq2y/Eek5+FO0kUOv6xYfjhp7MrDpAHAx0TElYXl15FOm9xcmN+alz8YeiUncScAv88x7wPMJJ3yuJ70XD9HGhNYbf17SR/+D+T1q57WKtSv9zpuzDEsIyUO/4eXk6GeBHYS6ZTlCtL/2pfpfXt5LumD/AFSz0/Pj4Ju9PuoIv47Sc/Jj1S44q6wvNpz/G1gHnAX6fTybYW4Kn2LdBp6DXAphf+5iPgLqXfnCNLpvY+w/v9kv4mIi0jvp9n5NNzdwHrHnOtW+9+s5xOkCwEWkNqmC1n3FHLRRGC+pKeA7wOHRUTPKdPK99UNpPFixfdZX17/1+bYniCdEr+O9J7r+WHfH/RiW9ZkPVdoWIfJycNxEfGrVseyMZQutX5FRPxrq2MxMzPrK/dwdaDc7f4mXj7d1VaUfutnf9I3cTMzs7bnhKvDSDqJdBrjqxFRedXQgJev+LkfuIk0bsnMzKzt+ZSimZmZWcncw2VmZmZWMidcZmZmZiVzwmVmZmZWMidcZmZmZiVzwmVmZmZWMidcZmZmZiVzwtUBJM2XtG+r4+hvkhZJek8/bessSbVuYYKkkLRbnv6BpP/oj/2amQHk+3HWu51Zb7b3UptVZdkRkm4ozD8lqaF7S1p5Nml1ANZ3EbF7M/Yj6SxgSUR8oxn7a5WI+GyrYzCz9UkKYExEdA/E7Q1UEbFVq2Mw93CZmZmZlc4JVwconnqTdKykCySdLenJfLpxfEXdoyUtkLRK0k8kbZGXrdMNnctC0m6SpgIfBb6Su6d/XSOWN0qaK2mlpPskHVpYdpak0yX9Jm/j95JeK+mUHMu9kvao2OTbqsWat/c+SXdIWi3pD5LeUli2h6Tb8nNwPrBFcaOSvixpmaSHJX2yYtlLpx97TgNI+pKk5XmdIwt1t5P0a0lPSLpF0rd7nkMlJ+f1npD0J0lv3tBradbpJL1J0rX5fTtf0gcKy66V9KnC/BGF99P1ufjO3H58pPD+/Jqkx3L79tGN3V6NeD8p6Z7cBl0haefCspD0z5IW5rbmeEm75vboidwWb1axvVqxbi7pvyX9r6RHlYY2vLKwfENt1naS5uR93gzsWrG8OGTiLEmnSbo0x3yTpF0LdSfktntNbq+v63kOlT4LrsvLHsttqzXICVdn+gAwGxgKzAH+p2L5R0k3h94VeD1Q9xRhRMwAzgH+KyK2ioj3V9aR9CpgLnAu8BrgMOB0SWML1Q7N+9seeB74I3Bbnr8Q+F4jsebEbCbwGWA74IfAnNxobQb8CvgpsC3wc+BDhTgnAv8OvBcYA9QbJ/ZaYBtgBDAFOE3SsLzsNODpXGdyfvSYALwzx71NPvbH6+zLrGNJ2hT4Nel+r68B/gU4R9Ib6q0bEe/Mk2/NbVDPh/1rSe3HCNL7b0Yft1eMdxLwNeCDQBfwO+C8imr7A38L7AN8BZgBfAwYBbwZOLxQd0OxnkhqK8YBu+U638xx1GuzTgOeA3YAPpkfG3IY8C1gGNANnJD309MOH01qV+8D/k9hveNJr90wYCTw/+rsxwqccHWmGyLisoh4gZR0vLVi+f9ExOKIWEl6ox2+3hY2zvuARRHxk4hYGxG3A78APlyoc1FE3BoRzwEXAc9FxNk51vOByh6uWrFOBX4YETdFxAsRMYuUwO2TH5sCp0TEXyPiQuCWwjYPBX4SEXdHxNPAsXWO66/AcXlblwFPAW+QNISUyB0TEc9ExAJgVsV6WwNvJN239J6IWFZnX2adbB9gK+DEiPhLRFwNXELf26D/iIjnI+I64FLSe7w/fBb4z/zeXQt8BxhX7OUifQl9IiLmA3cDv42IByJiDfAb1m/T1otVkkht2r9FxMqIeDLv67C8Ts02q9AOfTMino6Iu1m3Harmooi4OR/TOaQkD+BAYH5E/DIvOxV4pLDeX4GdgR0j4rmIWOeMiG2YE67OVHyDPANsIal4gcTiwvRDwI79tN+dgb3zqYLVklaTeqheW6jzaGH62SrzlYM7a8W6M/Clin2Nyst3BJbGundmf6gwvWOV7W7I47nx6fFMjrOLdOFJcVsvTecPk/8hfftcLmmGpFfX2ZdZJ9sRWBwRLxbKHiL15mysVTkJKW6vP9u07xfamJWAWDfe3rRptWLtArYEbi3s6/JcDhtus6q1Q/XatMrPiJ4Y19lPbkOLV1Z+hXT8N+fTwfV60qzACdfgNKowvRPwcJ5+mvSmB0BSMVECCDZsMXBdRAwtPLaKiM+VEOti4ISKfW0ZEecBy4AR+Vtjcd0ey6psd2OsANaSutarxUtEnBoRfwuMJZ0u+PJG7susEzwMjJJU/OzZCViap9dpg1j3y1otw/JwhuL2qrZpDW6vaDHwmYp25pUR8YdebqderI+RkrPdC/vZpnB14YbarJ52qD/atGUU2rPchr40HxGPRMSnI2JH0nCO01XjpylsfU64BqdpkkZK2hb4OulUHsCdwO6SxikNTj+2Yr1HgQ39lsslwOslfVzSpvnxNklvKiHWHwGflbS3kldJOkjS1qRxYWuBz+cYPgjsVdjmBcARksZK2hI4ZmMCy6dBfwkcK2lLSW8EPtGzPB/73nncytOkMRYvVt+a2aBwE6lH5Sv5vbkv8H7SmFOAO4AP5vfTbqQxk0W12qBvSdpM0t+Thjb8vI/b6/ED4GhJuwNI2kbShzdQvxHrxZp7/H4EnCzpNXlfIyTtn9ep2WZVaYfGsu5Y0t64FPgbSQfnsyLTKCSpkj4sqScBW0X6Eu42rUFOuAanc0kDHx8A7ge+DRARfwaOA64EFgKV5+fPBMbmLu9fVW40jzuYQBp38DCp2/okYPMSYp0HfJp0ym4VaeDnEXnZX0iDXI8gnQL4CKlB6onzN8ApwNV5vav7EN9RpAHxj5DGy51HGksG8GpSI7qK1MX/OPB/+7Avs7aW35vvBw4g9eqcDnwiIu7NVU4G/kJKhGaRxhcVHQvMym1QzzitR0jvsYdz/c/2cXvFeC8itWGzJT1BGqN1QO+P/CUbivWrpPboxryvK4E35DjqtVlHkU4LPgKcBfxkY4KLiMdIY27/i9RejQXm8XKb9jbgJklPkS7I+kJEPLAx+xqMtO4wF+t0khYBn4qIK1sdSyeSdBLw2ojY2G+YZtag3EP2s4gYWa+u9V4+9bsE+GhEXNPqeNqde7jM+kDpd8fekk9r7kU6ZXFRq+MyM9sYkvaXNFTS5qSfxBBwY4vD6gi+tY9Z32xNOo24I+m0xXeBi1sakZnZxns7aSjHZsAC4OCIeLa1IXUGn1I0MzMzK5lPKZqZmZmVzAmXmZmZWckG9Biu7bffPkaPHt3qMMysiW699dbHIqKrfs2Bz22Y2eCyofZrQCdco0ePZt68ea0Ow8yaSFK925K0DbdhZoPLhtovn1I0MzMzK5kTLjMzM7OSOeEyMzMzK5kTLjMzM7OSOeEyMzMzK5kTLjMzM7OSOeEyMzMzK5kTLjMzM7OSOeEyMzMzK5kTLjMzM7OSOeEyMzMzK9mAvpfiQDd6+qXrzC868aAWRWJmNrC5vbTBzj1cZmZmZiVzwmVmZmZWMidcZmZmZiVzwmVmZmZWMidcZmZmZiVzwmVmHU/SEEm3S7okz+8i6SZJ3ZLOl7RZLt88z3fn5aML2zg6l98naf/WHImZtSsnXGY2GHwBuKcwfxJwckTsBqwCpuTyKcCqXH5yroekscBhwO7AROB0SUOaFLuZdQAnXGbW0SSNBA4CfpznBbwbuDBXmQUcnKcn5Xny8v1y/UnA7Ih4PiIeBLqBvZpzBGbWCZxwmVmnOwX4CvBint8OWB0Ra/P8EmBEnh4BLAbIy9fk+i+VV1nHzKwuJ1xm1rEkvQ9YHhG3NnGfUyXNkzRvxYoVzdqtmQ1wDSVckoZKulDSvZLukfR2SdtKmitpYf47LNeVpFPz4NK7JO1Z2M7kXH+hpMllHZSZWfYO4AOSFgGzSacSvw8MldRza7ORwNI8vRQYBZCXbwM8Xiyvss46ImJGRIyPiPFdXV39ezRm1rYavZfi94HLI+KQfDXPlsDXgKsi4kRJ04HpwFeBA4Ax+bE3cAawt6RtgWOA8UAAt0qaExGr+vWIzMyyiDgaOBpA0r7Av0fERyX9HDiElIRNBi7Oq8zJ83/My6+OiJA0BzhX0veAHUnt283NPJZ2UnnfRDNroIdL0jbAO4EzASLiLxGxmnUHl1YOOj07khtJ3yR3APYH5kbEypxkzSVd7WNm1mxfBb4oqZs0RuvMXH4msF0u/yLpiyQRMR+4AFgAXA5Mi4gXmh61mbWtRnq4dgFWAD+R9FbgVtIl1sMjYlmu8wgwPE/XGlzqQadm1jIRcS1wbZ5+gCpXGUbEc8CHa6x/AnBCeRGaWSdrZAzXJsCewBkRsQfwNPlbX4+ICNJpwj7zgFMzMzPrNI0kXEuAJRFxU56/kJSAPZpPFZL/Ls/Law0ubWjQqQecmpmZWaepm3BFxCPAYklvyEX7kcYx9AwuhfUHnX4iX624D7Amn3q8ApggaVi+onFCLjMzMzPraI1epfgvwDn5CsUHgCNJydoFkqYADwGH5rqXAQeSfon5mVyXiFgp6XjgllzvuIhY2S9HYWZmZjaANZRwRcQdpJ9zqLRflboBTKuxnZnAzN4EaGZmZtbu/EvzZmZmZiVzwmVmZmZWMidcZmZmZiVzwmVmZmZWMidcZmZmZiVzwmVmZmZWMidcZmZmZiVzwmVmZmZWMidcZmZmZiVzwmVmZmZWskbvpWhmZtZvRk+/dL2yRSce1IJIzJrDPVxmZmZmJXPCZWZmZlYyJ1xmZmZmJXPCZWZmZlYyJ1xmZmZmJXPCZWZmZlYyJ1xm1rEkbSHpZkl3Spov6Vu5/CxJD0q6Iz/G5XJJOlVSt6S7JO1Z2NZkSQvzY3KrjsnM2pN/h8vMOtnzwLsj4ilJmwI3SPpNXvbliLiwov4BwJj82Bs4A9hb0rbAMcB4IIBbJc2JiFVNOQoza3tOuPqRf8jPbGCJiACeyrOb5kdsYJVJwNl5vRslDZW0A7AvMDciVgJImgtMBM4rK3Yz6yw+pWhmHU3SEEl3AMtJSdNNedEJ+bThyZI2z2UjgMWF1ZfkslrlZmYNccJlZh0tIl6IiHHASGAvSW8GjgbeCLwN2Bb4an/tT9JUSfMkzVuxYkV/bdbM2pwTLjMbFCJiNXANMDEilkXyPPATYK9cbSkwqrDayFxWq7zafmZExPiIGN/V1dXfh2FmbcoJl5l1LEldkobm6VcC7wXuzeOykCTgYODuvMoc4BP5asV9gDURsQy4ApggaZikYcCEXGZm1hAPmm9QtQHxZjbg7QDMkjSE9AXzgoi4RNLVkroAAXcAn831LwMOBLqBZ4AjASJipaTjgVtyveN6BtCbmTXCCZeZdayIuAvYo0r5u2vUD2BajWUzgZn9GqCZDRo+pWhmZmZWMidcZmZmZiXzKUUzM9toHt9q1hj3cJmZmZmVzAmXmZmZWcmccJmZmZmVrKGES9IiSX+SdIekeblsW0lzJS3Mf4flckk6VVJ3vk/ZnoXtTM71F0qaXM4hmZmZmQ0svenheldEjIuI8Xl+OnBVRIwBrsrzAAcAY/JjKnAGpAQNOAbYm3QbjWN6kjQzMzOzTtaXU4qTgFl5ehbp9hg95Wfn+5TdCAzNt9HYH5gbESsjYhUwF5jYh/2bmZmZtYVGE64AfivpVklTc9nwfI8xgEeA4Xl6BLC4sO6SXFarfB2SpkqaJ2neihUrGgzPzMzMbOBq9He4/i4ilkp6DTBX0r3FhRERkqI/AoqIGcAMgPHjx/fLNs3MzMxaqaEerohYmv8uBy4ijcF6NJ8qJP9dnqsvBUYVVh+Zy2qVm5mZmXW0ugmXpFdJ2rpnGpgA3A3MAXquNJwMXJyn5wCfyFcr7gOsyacerwAmSBqWB8tPyGVmZmZmHa2RU4rDgYsk9dQ/NyIul3QLcIGkKcBDwKG5/mXAgUA38AxwJEBErJR0PHBLrndcRKzstyMxMzMzG6DqJlwR8QDw1irljwP7VSkPYFqNbc0EZvY+TDMzM7P25V+aNzMzMyuZEy4zMzOzkjnhMjMzMyuZEy4zMzOzkjnhMjMzMyuZEy4zMzOzkjnhMjMzMytZo/dSNDNrO5K2AK4HNie1dxdGxDGSdgFmA9sBtwIfj4i/SNocOBv4W+Bx4CMRsShv62hgCvAC8PmI8J0y+tno6ZeuM7/oxINaFIlZ/3MPl5l1sueBd0fEW4FxwMR8y7GTgJMjYjdgFSmRIv9dlctPzvWQNBY4DNgdmAicLmlIU4/EzNqaEy4z61iRPJVnN82PAN4NXJjLZwEH5+lJeZ68fD+l+5pNAmZHxPMR8SDp1mV7NeEQzKxDOOEys44maYikO4DlwFzgfmB1RKzNVZYAI/L0CGAxQF6+hnTa8aXyKuuYmdXlhMvMOlpEvBAR44CRpF6pN5a5P0lTJc2TNG/FihVl7srM2ogTLjMbFCJiNXAN8HZgqKSei4ZGAkvz9FJgFEBevg1p8PxL5VXWqdzPjIgYHxHju7q6+v04zKw9OeEys44lqUvS0Dz9SuC9wD2kxOuQXG0ycHGenpPnycuvjojI5YdJ2jxf4TgGuLk5R2FmncA/C2FmnWwHYFa+ovAVwAURcYmkBcBsSd8GbgfOzPXPBH4qqRtYSboykYiYL+kCYAGwFpgWES80+VjMrI054TKzjhURdwF7VCl/gCpXGUbEc8CHa2zrBOCE/o7RzAYHn1I0MzMzK5kTLjMzM7OSOeEyMzMzK5kTLjMzM7OSOeEyMzMzK5kTLjMzM7OSOeEyMzMzK5kTLjMzM7OSOeEyMzMzK5kTLjMzM7OSOeEyMzMzK5kTLjMzM7OSOeEyMzMzK5kTLjMzM7OSOeEyMzMzK1nDCZekIZJul3RJnt9F0k2SuiWdL2mzXL55nu/Oy0cXtnF0Lr9P0v79fTBmZmZmA1Fveri+ANxTmD8JODkidgNWAVNy+RRgVS4/OddD0ljgMGB3YCJwuqQhfQvfzMzMbOBrKOGSNBI4CPhxnhfwbuDCXGUWcHCenpTnycv3y/UnAbMj4vmIeBDoBvbqj4MwMzMzG8ga7eE6BfgK8GKe3w5YHRFr8/wSYESeHgEsBsjL1+T6L5VXWcfMzMysY9VNuCS9D1geEbc2IR4kTZU0T9K8FStWNJZ0nW8AABXYSURBVGOXZmZmZqVqpIfrHcAHJC0CZpNOJX4fGCppk1xnJLA0Ty8FRgHk5dsAjxfLq6zzkoiYERHjI2J8V1dXrw/IzMzMbKCpm3BFxNERMTIiRpMGvV8dER8FrgEOydUmAxfn6Tl5nrz86oiIXH5YvopxF2AMcHO/HYmZmZnZANWX3+H6KvBFSd2kMVpn5vIzge1y+ReB6QARMR+4AFgAXA5Mi4gX+rB/M7MNkjRK0jWSFkiaL+kLufxYSUsl3ZEfBxbWqfrzNZIm5rJuSdNbcTxm1r42qV/lZRFxLXBtnn6AKlcZRsRzwIdrrH8CcEJvgzQz20hrgS9FxG2StgZulTQ3Lzs5Iv67WLni52t2BK6U9Pq8+DTgvaQLfm6RNCciFjTlKMys7fUq4TIzaycRsQxYlqeflHQPG746+qWfrwEezD31PV8su/MXTSTNznWdcJlZQ3xrHzMbFPJdL/YAbspFR0m6S9JMScNyWa2fr/HP2phZnzjhMrOOJ2kr4BfAv0bEE8AZwK7AOFIP2Hf7cV/+aRszW48TLjPraJI2JSVb50TELwEi4tGIeCEiXgR+xMunDWv9fE1DP2uTt+2ftjGz9XgMl5l1rHxbsTOBeyLie4XyHfL4LoB/BO7O03OAcyV9jzRovufnawSMyT9ps5Q0sP6fmnMUA8vo6Ze2OgSztuSEy8w62TuAjwN/knRHLvsacLikcUAAi4DPQPr5Gkk9P1+zlsLP10g6CrgCGALMzD91Y2bWECdcZtaxIuIGUu9Upcs2sE7Vn6+JiMs2tJ6Z2YZ4DJeZmZlZyZxwmZmZmZXMCZeZmZlZyZxwmZmZmZXMCZeZmZlZyZxwmZmZmZXMPwtRssofCVx04kEtisTMzMxaxT1cZmZmZiVzwmVmZmZWMidcZmZmZiVzwmVmZmZWMidcZmZmZiVzwmVmZmZWMidcZmZmZiVzwmVmZmZWMidcZmZmZiVzwmVmZmZWMt/ax8zMBqTKW6OBb49m7cs9XGZmZmYlc8JlZmZmVjInXGZmZmYlc8JlZmZmVjInXGbWsSSNknSNpAWS5kv6Qi7fVtJcSQvz32G5XJJOldQt6S5Jexa2NTnXXyhpcquOyczakxMuM+tka4EvRcRYYB9gmqSxwHTgqogYA1yV5wEOAMbkx1TgDEgJGnAMsDewF3BMT5JmZtYIJ1xm1rEiYllE3JannwTuAUYAk4BZudos4OA8PQk4O5IbgaGSdgD2B+ZGxMqIWAXMBSY28VDMrM3VTbgkbSHpZkl35i75b+XyXSTdlLvez5e0WS7fPM935+WjC9s6OpffJ2n/sg7KzKxSbov2AG4ChkfEsrzoEWB4nh4BLC6stiSX1So3M2tIIz1czwPvjoi3AuOAiZL2AU4CTo6I3YBVwJRcfwqwKpefnOuRu/EPA3YnfTM8XdKQ/jwYM7NqJG0F/AL414h4orgsIgKIftzXVEnzJM1bsWJFf23WzNpc3YQrd60/lWc3zY8A3g1cmMsru+R7uuovBPaTpFw+OyKej4gHgW7SWAgzs9JI2pSUbJ0TEb/MxY/mU4Xkv8tz+VJgVGH1kbmsVvl6ImJGRIyPiPFdXV39dyBm1tYaurVP7om6FdgNOA24H1gdEWtzlWL3+ktd7xGxVtIaYLtcfmNhswO2S77a7STMrP3kL3tnAvdExPcKi+YAk4ET89+LC+VHSZpNGiC/JiKWSboC+E5hoPwE4OhmHIOZdYaGEq6IeAEYJ2kocBHwxrICkjSVdHUQO+20U1m7MbPB4R3Ax4E/Sbojl32NlGhdIGkK8BBwaF52GXAgqQf+GeBIgIhYKel44JZc77iIWNmcQzCzTtCrm1dHxGpJ1wBvJ129s0nu5Sp2r/d0vS+RtAmwDfA4DXbJR8QMYAbA+PHj+21chZkNPhFxA6Aai/erUj+AaTW2NROY2X/Rmdlg0shVil25ZwtJrwTeS7q0+hrgkFytsku+50cBDwGuzo3YHOCwfBXjLqTfubm5vw7EzMzMbKBqpIdrB2BWHsf1CuCCiLhE0gJgtqRvA7eTxkmQ//5UUjewknRlIhExX9IFwALSjxFOy6cqzczMzDpa3YQrIu4i/XZNZfkDVLnKMCKeAz5cY1snACf0PkwzMzOz9uVfmjczMzMrmRMuMzMzs5I54TIzMzMrmRMuMzMzs5I54TIzMzMrmRMuMzMzs5I54TIzMzMrmRMuMzMzs5I54TIzMzMrmRMuMzMzs5I54TIzMzMrmRMuMzMzs5I54TIzMzMrmRMuMzMzs5I54TIzMzMrmRMuMzMzs5I54TIzMzMr2SatDsDMzAam0dMvbXUIZh3DPVxmZmZmJXPCZWZmZlYyJ1xm1tEkzZS0XNLdhbJjJS2VdEd+HFhYdrSkbkn3Sdq/UD4xl3VLmt7s4zCz9uaEy8w63VnAxCrlJ0fEuPy4DEDSWOAwYPe8zumShkgaApwGHACMBQ7Pdc3MGuJB82bW0SLiekmjG6w+CZgdEc8DD0rqBvbKy7oj4gEASbNz3QX9HK6ZdSj3cJnZYHWUpLvyKcdhuWwEsLhQZ0kuq1W+HklTJc2TNG/FihVlxG1mbcgJl5kNRmcAuwLjgGXAd/trwxExIyLGR8T4rq6u/tqsmbU5n1I0s0EnIh7tmZb0I+CSPLsUGFWoOjKXsYFyM7O63MNlZoOOpB0Ks/8I9FzBOAc4TNLmknYBxgA3A7cAYyTtImkz0sD6Oc2M2czam3u4zKyjSToP2BfYXtIS4BhgX0njgAAWAZ8BiIj5ki4gDYZfC0yLiBfydo4CrgCGADMjYn6TD8XM2pgTLjPraBFxeJXiMzdQ/wTghCrllwGX9WNoZjaI+JSimZmZWcncw2VmZm2j2g21F514UAsiMesd93CZmZmZlaxuwiVplKRrJC2QNF/SF3L5tpLmSlqY/w7L5ZJ0ar7f2F2S9ixsa3Kuv1DS5PIOy8zMzGzgaKSHay3wpYgYC+wDTMv3EJsOXBURY4Cr8jyke42NyY+ppB8YRNK2pKuD9ibdKuOYwq87m5mZmXWsuglXRCyLiNvy9JPAPaRbWkwCZuVqs4CD8/Qk4OxIbgSG5t+82R+YGxErI2IVMJfqN5Q1MzMz6yi9GsOVbwC7B3ATMDwiluVFjwDD83Sf7kXm+5CZmZlZp2k44ZK0FfAL4F8j4onisogI0g8I9pnvQ2ZmZmadpqGES9KmpGTrnIj4ZS5+tOf2GPnv8lxe615kG7pHmZmZmVnHqvs7XJJE+lXmeyLie4VFc4DJwIn578WF8qMkzSYNkF8TEcskXQF8pzBQfgJwdP8cRvvwb8iYmZkNPo388Ok7gI8Df5J0Ry77GinRukDSFOAh4NC87DLgQKAbeAY4EiAiVko6nnQTWIDjImJlvxyFmZmZ2QBWN+GKiBsA1Vi8X5X6AUyrsa2ZwMzeBGhmZmbW7vxL82ZmZmYlc8JlZmZmVjInXGZmZmYlc8JlZmZmVjInXGZmZmYlc8JlZmZmVjInXGZmZmYlc8JlZmZmVjInXGZmZmYlc8JlZmZmVjInXGbW0STNlLRc0t2Fsm0lzZW0MP8dlssl6VRJ3ZLukrRnYZ3Juf5CSZNbcSxm1r6ccJlZpzsLmFhRNh24KiLGAFfleYADgDH5MRU4A1KCBhwD7A3sBRzTk6SZmTXCCZeZdbSIuB5YWVE8CZiVp2cBBxfKz47kRmCopB2A/YG5EbEyIlYBc1k/iTMzq8kJl5kNRsMjYlmefgQYnqdHAIsL9ZbkslrlZmYNccJlZoNaRAQQ/bU9SVMlzZM0b8WKFf21WTNrc064zGwwejSfKiT/XZ7LlwKjCvVG5rJa5euJiBkRMT4ixnd1dfV74GbWnpxwmdlgNAfoudJwMnBxofwT+WrFfYA1+dTjFcAEScPyYPkJuczMrCGbtDoAM7MySToP2BfYXtIS0tWGJwIXSJoCPAQcmqtfBhwIdAPPAEcCRMRKSccDt+R6x0VE5UB8M7OanHCZWUeLiMNrLNqvSt0AptXYzkxgZj+GZmaDiE8pmpmZmZXMCZeZmZlZyZxwmZmZmZXMCZeZmZlZyZxwmZmZmZXMVymamRmjp1/a6hDMOpp7uMzMzMxK5oTLzMzMrGQ+pYi70s3MzKxcTrjMzKytVX5pXnTiQS2KxKw2n1I0MzMzK5kTLjMzM7OS1U24JM2UtFzS3YWybSXNlbQw/x2WyyXpVEndku6StGdhncm5/kJJk8s5HDMzM7OBp5EerrOAiRVl04GrImIMcFWeBzgAGJMfU4EzICVowDHA3sBewDE9SZqZmZlZp6ubcEXE9cDKiuJJwKw8PQs4uFB+diQ3AkMl7QDsD8yNiJURsQqYy/pJnJmZmVlH2tgxXMMjYlmefgQYnqdHAIsL9ZbkslrlZmZmZh2vz4PmIyKA6IdYAJA0VdI8SfNWrFjRX5s1MzMza5mNTbgezacKyX+X5/KlwKhCvZG5rFb5eiJiRkSMj4jxXV1dGxmemZmZ2cCxsQnXHKDnSsPJwMWF8k/kqxX3AdbkU49XABMkDcuD5SfkMjMzM7OOV/eX5iWdB+wLbC9pCelqwxOBCyRNAR4CDs3VLwMOBLqBZ4AjASJipaTjgVtyveMionIg/qBV7dZC/qVkMzOzzlE34YqIw2ss2q9K3QCm1djOTGBmr6IzMzMz6wD+pXkzMzOzkjnhMrNBS9IiSX+SdIekebms13fSMDOrxwmXmQ1274qIcRExPs/36k4aZmaNcMJlZrau3t5Jw8ysLidcZjaYBfBbSbdKmprLensnDTOzuupepWhm1sH+LiKWSnoNMFfSvcWFERGSenUnjZy4TQXYaaed+i9SM2tr7uEys0ErIpbmv8uBi4C96P2dNCq36btlmNl6nHCZ2aAk6VWStu6ZJt0B4256fycNM7O6fErRzAar4cBFkiC1hedGxOWSbqEXd9IwM2uEEy4zG5Qi4gHgrVXKH6eXd9KwgcW3S7OByKcUzczMzErmHi4zs0GoWi+QmZXHPVxmZmZmJXPCZWZmZlYyJ1xmZmZmJXPCZWZmZlYyJ1xmZmZmJfNVigNU5RVE/g0ZMzOz9uUeLjMzM7OSOeEyMzMzK5kTLjMzM7OSOeEyMzMzK5kTLjMzM7OSDbqrFH3/MDMbbNzu+cpvaz33cJmZmZmVzAmXmZmZWckG3SnFdlXtlIC7xM3MzNqDe7jMzMzMSuaEy8zMzKxkPqVoZmaDjodpWLO5h8vMzMysZO7hamP+XRkzM7P20PSES9JE4PvAEODHEXFis2MwM9sY7dJ++YdOzQaepiZckoYApwHvBZYAt0iaExELytifGx0z6y/Nbr+s+Tyuy8rU7B6uvYDuiHgAQNJsYBLgBqsfuLEwK9WAbL/8xdKsPTQ74RoBLC7MLwH2bnIMg0ojjbGTMrOGtLz9cnLVfBv7nLtdtUoDbtC8pKnA1Dz7lKT78vT2wGOtiarX2iXW7YHHdFKrw6irrZ7PVgfRgIEe586tDqAvNtCGtcJAf62hQ2NsQbvaDs8jtEecfYmxZvvV7IRrKTCqMD8yl70kImYAMypXlDQvIsaXG17/aJdYHWf/cpwdr277BbXbsFZoh9faMfaPdogR2iPOsmJs9u9w3QKMkbSLpM2Aw4A5TY7BzGxjuP0ys43W1B6uiFgr6SjgCtJl1TMjYn4zYzAz2xhuv8ysL5o+hisiLgMu24hVB0QXfYPaJVbH2b8cZ4frQ/vVKu3wWjvG/tEOMUJ7xFlKjIqIMrZrZmZmZpnvpWhmZmZWsgGbcEnaVtJcSQvz32FV6oyT9EdJ8yXdJekjTYxvoqT7JHVLml5l+eaSzs/Lb5I0ulmxVcRRL84vSlqQn7+rJLXkkvx6cRbqfUhSSGrJVS6NxCnp0Pyczpd0brNjzDHUe913knSNpNvza39gK+K0/tVIu5nrXS5ptaRLmhjbgG8zG4jxnZJuk7RW0iHNjq/BGFvepjcQ42cl/UnSHZJukDS22TE2EmehXv987kTEgHwA/wVMz9PTgZOq1Hk9MCZP7wgsA4Y2IbYhwP3A64DNgDuBsRV1/hn4QZ4+DDi/Bc9hI3G+C9gyT39uoMaZ620NXA/cCIwfiHECY4DbgWF5/jUDNM4ZwOfy9FhgUbPj9KOU175uu5mX7Qe8H7ikSXEN+DazwRhHA28BzgYOacHrO+Db9AZjfHVh+gPA5QPxucz1+u1zZ8D2cJFumTErT88CDq6sEBF/joiFefphYDnQ1YTYXrrFR0T8Bei5xUdRMf4Lgf0kqQmxFdWNMyKuiYhn8uyNpN8WarZGnk+A44GTgOeaGVxBI3F+GjgtIlYBRMTyJscIjcUZwKvz9DbAw02Mz8pTt90EiIirgCebFRTt0WY20l4uioi7gBebGFdRO7TpjcT4RGH2VaT2qNma/rkzkBOu4RGxLE8/AgzfUGVJe5Gy1PvLDozqt/gYUatORKwF1gDbNSG2qjFk1eIsmgL8ptSIqqsbp6Q9gVER0cp7mzTyfL4eeL2k30u6UdLEpkX3skbiPBb4mKQlpKvu/qU5oVnJetVuNlE7tJm9bS9boR3a9IZilDRN0v2kXtnPNym2oqZ/7rT01j6SrgReW2XR14szERGSambAknYAfgpMjohWffNoa5I+BowH/qHVsVSS9Arge8ARLQ6lEZuQTivuS/pmeb2kv4mI1S2Nan2HA2dFxHclvR34qaQ3+/0z8PVXu2mdbSC36QARcRpwmqR/Ar4BTG5xSOso43OnpQlXRLyn1jJJj0raISKW5YSq6qkZSa8GLgW+HhE3lhRqpUZu8dFTZ4mkTUinbR5vTnjrxdCj6q1IJL2H1Fj/Q0Q836TYiurFuTXwZuDafIbhtcAcSR+IiHlNi7Kx53MJcFNE/BV4UNKfSQnYLc0JEWgszinARICI+KOkLUj3D2vFKVDrhf5oN1ugHdrMhtrLFmuHNr23z+Ns4IxSI6qu6Z87A/mU4hxezngnAxdXVlC6vcZFwNkRcWETY2vkFh/F+A8Bro48Aq+J6sYpaQ/gh8AHWjTeCOrEGRFrImL7iBgdEaNJ4xKanWzVjTP7Fal3C0nbk04xPtDMIGkszv8lDZxG0puALYAVTY3SylC33WyRdmgz2+HWTe3QpjcS45jC7EHAwibG16P5nzvNvjKg0Qfp3P1VpBfiSmDbXD4e+HGe/hjwV+COwmNck+I7EPgzaczY13PZcfkFgfQB9nOgG7gZeF2Lnsd6cV4JPFp4/uYMxDgr6l5LC65SbPD5FKkbegHwJ+CwARrnWOD3pCtz7gAmtCJOP/r9da/bbub535ES7GdJvbL7NyG2Ad9mNhDj2/Lz9TSp923+AIyx5W16AzF+H5if47sG2L3ZMTYSZ0XdPn/u+JfmzczMzEo2kE8pmpmZmXUEJ1xmZmZmJXPCZWZmZlYyJ1xmZmZmJXPCZWZmZlYyJ1xmZmZmJXPCZWZmZlYyJ1xmZmZmJfv/opBUxX0n5+sAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 720x360 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-_SkE1u8RGhi"
      },
      "source": [
        "### Top k most similar words\n",
        "\n",
        "Histograms show that the output embeddings have a long tail in both cases. Terms with high contextual similarity to the input ambiguous term are going to be located on this tail.\n",
        "\n",
        "Interestingly the length of the tail becomes shorter in both cases on the output embedding side (regressing to the mean as more information becomes available?).\n",
        "\n",
        "We find contextually similar words based on cosine similarity with the input words. As expected, output embeddings for `bank` in the first sentence show similarity with words related to banking, and output embeddings for `bank` in the second sentence show similarity with words related to river and shore."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MHQbTBlTNn9U",
        "outputId": "663bce01-43fc-47f0-b7c8-e6cedb40ddd8"
      },
      "source": [
        "def get_top_most_similar_words(cosims, k=10, show_scores=True, show_real_words=True):\n",
        "  top_k_idxs = np.argsort(cosims)[::-1][0:k+10]\n",
        "  tokens = tokenizer.convert_ids_to_tokens(top_k_idxs)\n",
        "  if show_scores:\n",
        "    top_scores = [cosims[i] for i in top_k_idxs]\n",
        "    if show_real_words:\n",
        "      return [(token, score) for token, score in zip(tokens, top_scores) if\n",
        "              not(token.startswith(\"[\")) and\n",
        "              not(token.startswith(\"##\")) and\n",
        "              len(token) > 1][0:k]\n",
        "    else:\n",
        "      return [(token, score) for token, score in zip(tokens, top_scores)][0:k]\n",
        "  else:\n",
        "    if show_real_words:\n",
        "      return [token for token in tokens if \n",
        "              not(token.startswith(\"[\")) and \n",
        "              not(token.startswith(\"##\")) and \n",
        "              len(token) > 1][0:k]\n",
        "    else:\n",
        "      return tokens[0:k]\n",
        "\n",
        "\n",
        "get_top_most_similar_words(cosims_in_0)"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('bank', 0.9322858452796936),\n",
              " ('banks', 0.6398346424102783),\n",
              " ('Bank', 0.5244991183280945),\n",
              " ('banking', 0.42404094338417053),\n",
              " ('banker', 0.3195374608039856),\n",
              " ('Banking', 0.27450740337371826),\n",
              " ('Banks', 0.2625081241130829),\n",
              " ('shore', 0.24294957518577576),\n",
              " ('hotel', 0.22805191576480865),\n",
              " ('store', 0.22024661302566528)]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "b6J2HmVHYUU_",
        "outputId": "05880251-bec2-4a62-ddd0-c5254553530c"
      },
      "source": [
        "get_top_most_similar_words(cosims_out_0)"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('bank', 0.18816906213760376),\n",
              " ('banker', 0.17837834358215332),\n",
              " ('her', 0.13340264558792114),\n",
              " ('banking', 0.13269563019275665),\n",
              " ('Bank', 0.12636975944042206),\n",
              " ('car', 0.1258864849805832),\n",
              " ('Banking', 0.12153404206037521),\n",
              " ('and', 0.11937716603279114),\n",
              " ('building', 0.11740049719810486),\n",
              " ('the', 0.11648747324943542)]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GMYKblvyR89R",
        "outputId": "9d577b68-a71f-40e2-d1ca-863084597f3e"
      },
      "source": [
        "get_top_most_similar_words(cosims_in_1)"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('bank', 0.9364502429962158),\n",
              " ('banks', 0.644376814365387),\n",
              " ('Bank', 0.5293071269989014),\n",
              " ('banking', 0.42480504512786865),\n",
              " ('banker', 0.32117369771003723),\n",
              " ('Banking', 0.2746722102165222),\n",
              " ('Banks', 0.26500627398490906),\n",
              " ('shore', 0.23942628502845764),\n",
              " ('hotel', 0.23093190789222717),\n",
              " ('store', 0.22488842904567719)]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fjBqIQ6NSDFu",
        "outputId": "07042770-6847-4e62-dd48-8b248d862f1c"
      },
      "source": [
        "get_top_most_similar_words(cosims_out_1)"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('river', 0.15005742013454437),\n",
              " ('bank', 0.14677192270755768),\n",
              " ('shore', 0.14008678495883942),\n",
              " ('the', 0.13924647867679596),\n",
              " ('running', 0.1298357993364334),\n",
              " ('side', 0.128604456782341),\n",
              " ('shoreline', 0.12534601986408234),\n",
              " ('deer', 0.12421098351478577),\n",
              " ('ran', 0.12203148007392883),\n",
              " ('one', 0.12113437801599503)]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rZi4yRLEUCAk"
      },
      "source": [
        ""
      ],
      "execution_count": 21,
      "outputs": []
    }
  ]
}