ARC_dataset_Tagrec_BERT_USE_final.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"accelerator":"GPU","colab":{"name":"ARC_dataset_Tagrec_BERT_USE_final.ipynb","provenance":[],"collapsed_sections":[],"toc_visible":true},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.7"}},"cells":[{"cell_type":"code","metadata":{"id":"sR9av2JU3kf6","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625202647056,"user_tz":-330,"elapsed":691,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"5138f138-d896-467a-e12c-bc7a2ce7fca3"},"source":["import os\n","os.environ['CUDA_LAUNCH_BLOCKING'] = \"1\"\n","import torch\n","import logging\n","logging.basicConfig(level=logging.ERROR)\n","# If there's a GPU available...\n","if torch.cuda.is_available():    \n","\n","    # Tell PyTorch to use the GPU.    \n","    device = torch.device(\"cuda\")\n","\n","    print('There are %d GPU(s) available.' % torch.cuda.device_count())\n","\n","    print('We will use the GPU:', torch.cuda.get_device_name(0))\n","\n","# If not...\n","else:\n","    print('No GPU available, using the CPU instead.')\n","    device = torch.device(\"cpu\")"],"execution_count":null,"outputs":[{"output_type":"stream","text":["There are 1 GPU(s) available.\n","We will use the GPU: Tesla T4\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"dvJuX6zEoC8M","executionInfo":{"status":"ok","timestamp":1625202696635,"user_tz":-330,"elapsed":49111,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"acef77ab-cb4e-4f8d-9bc6-744a28a4f5c3"},"source":["!pip install tensorflow==1.13.1\n","! pip install tensorflow-hub==0.7.0"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Collecting tensorflow==1.13.1\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d4/29/6b4f1e02417c3a1ccc85380f093556ffd0b35dc354078074c5195c8447f2/tensorflow-1.13.1-cp37-cp37m-manylinux1_x86_64.whl (92.6MB)\n","\u001b[K     |████████████████████████████████| 92.6MB 43kB/s \n","\u001b[?25hRequirement already satisfied: numpy>=1.13.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.13.1) (1.19.5)\n","Requirement already satisfied: gast>=0.2.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.13.1) (0.4.0)\n","Collecting tensorflow-estimator<1.14.0rc0,>=1.13.0\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/bb/48/13f49fc3fa0fdf916aa1419013bb8f2ad09674c275b4046d5ee669a46873/tensorflow_estimator-1.13.0-py2.py3-none-any.whl (367kB)\n","\u001b[K     |████████████████████████████████| 368kB 50.3MB/s \n","\u001b[?25hRequirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.13.1) (1.1.2)\n","Requirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.13.1) (3.12.4)\n","Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.13.1) (1.1.0)\n","Collecting keras-applications>=1.0.6\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/71/e3/19762fdfc62877ae9102edf6342d71b28fbfd9dea3d2f96a882ce099b03f/Keras_Applications-1.0.8-py3-none-any.whl (50kB)\n","\u001b[K     |████████████████████████████████| 51kB 9.1MB/s \n","\u001b[?25hRequirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.13.1) (1.15.0)\n","Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.13.1) (0.36.2)\n","Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.13.1) (1.34.1)\n","Collecting tensorboard<1.14.0,>=1.13.0\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/0f/39/bdd75b08a6fba41f098b6cb091b9e8c7a80e1b4d679a581a0ccd17b10373/tensorboard-1.13.1-py3-none-any.whl (3.2MB)\n","\u001b[K     |████████████████████████████████| 3.2MB 42.8MB/s \n","\u001b[?25hRequirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.13.1) (0.8.1)\n","Requirement already satisfied: absl-py>=0.1.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow==1.13.1) (0.12.0)\n","Collecting mock>=2.0.0\n","  Downloading https://files.pythonhosted.org/packages/5c/03/b7e605db4a57c0f6fba744b11ef3ddf4ddebcada35022927a2b5fc623fdf/mock-4.0.3-py3-none-any.whl\n","Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from protobuf>=3.6.1->tensorflow==1.13.1) (57.0.0)\n","Requirement already satisfied: h5py in /usr/local/lib/python3.7/dist-packages (from keras-applications>=1.0.6->tensorflow==1.13.1) (3.1.0)\n","Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard<1.14.0,>=1.13.0->tensorflow==1.13.1) (3.3.4)\n","Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard<1.14.0,>=1.13.0->tensorflow==1.13.1) (1.0.1)\n","Requirement already satisfied: cached-property; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from h5py->keras-applications>=1.0.6->tensorflow==1.13.1) (1.5.2)\n","Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard<1.14.0,>=1.13.0->tensorflow==1.13.1) (4.5.0)\n","Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->markdown>=2.6.8->tensorboard<1.14.0,>=1.13.0->tensorflow==1.13.1) (3.7.4.3)\n","Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->markdown>=2.6.8->tensorboard<1.14.0,>=1.13.0->tensorflow==1.13.1) (3.4.1)\n","\u001b[31mERROR: kapre 0.3.5 has requirement tensorflow>=2.0.0, but you'll have tensorflow 1.13.1 which is incompatible.\u001b[0m\n","Installing collected packages: mock, tensorflow-estimator, keras-applications, tensorboard, tensorflow\n","  Found existing installation: tensorflow-estimator 2.5.0\n","    Uninstalling tensorflow-estimator-2.5.0:\n","      Successfully uninstalled tensorflow-estimator-2.5.0\n","  Found existing installation: tensorboard 2.5.0\n","    Uninstalling tensorboard-2.5.0:\n","      Successfully uninstalled tensorboard-2.5.0\n","  Found existing installation: tensorflow 2.5.0\n","    Uninstalling tensorflow-2.5.0:\n","      Successfully uninstalled tensorflow-2.5.0\n","Successfully installed keras-applications-1.0.8 mock-4.0.3 tensorboard-1.13.1 tensorflow-1.13.1 tensorflow-estimator-1.13.0\n","Collecting tensorflow-hub==0.7.0\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/00/0e/a91780d07592b1abf9c91344ce459472cc19db3b67fdf3a61dca6ebb2f5c/tensorflow_hub-0.7.0-py2.py3-none-any.whl (89kB)\n","\u001b[K     |████████████████████████████████| 92kB 5.9MB/s \n","\u001b[?25hRequirement already satisfied: numpy>=1.12.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow-hub==0.7.0) (1.19.5)\n","Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow-hub==0.7.0) (1.15.0)\n","Requirement already satisfied: protobuf>=3.4.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow-hub==0.7.0) (3.12.4)\n","Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from protobuf>=3.4.0->tensorflow-hub==0.7.0) (57.0.0)\n","Installing collected packages: tensorflow-hub\n","  Found existing installation: tensorflow-hub 0.12.0\n","    Uninstalling tensorflow-hub-0.12.0:\n","      Successfully uninstalled tensorflow-hub-0.12.0\n","Successfully installed tensorflow-hub-0.7.0\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"pdc9-F6KbJym"},"source":["# Data preparation"]},{"cell_type":"code","metadata":{"id":"da-zaZJUGMFU","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625202696636,"user_tz":-330,"elapsed":18,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"1d1ffad7-aab0-4228-cd92-f113cfb17219"},"source":["#No need to mount if you download model and data to local and load it\n","from google.colab import drive\n","drive.mount('/content/drive')"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"PlE6vskY9VmG"},"source":["# no need to execute if model is downloaded to local\n","!cp -r \"/content/drive/My Drive/research_lo_content_taxonomy_classification/model_euclidean_USE_cos_QC/\" /content/"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"BzKeqoCs3kgA","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625202700371,"user_tz":-330,"elapsed":2416,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"2af8812d-201e-41cb-bd4b-307992e1bf40"},"source":["!pip install transformers==2.8.0"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Requirement already satisfied: transformers==2.8.0 in /usr/local/lib/python3.7/dist-packages (2.8.0)\n","Requirement already satisfied: boto3 in /usr/local/lib/python3.7/dist-packages (from transformers==2.8.0) (1.17.104)\n","Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers==2.8.0) (3.0.12)\n","Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers==2.8.0) (2019.12.20)\n","Requirement already satisfied: sacremoses in /usr/local/lib/python3.7/dist-packages (from transformers==2.8.0) (0.0.45)\n","Requirement already satisfied: sentencepiece in /usr/local/lib/python3.7/dist-packages (from transformers==2.8.0) (0.1.96)\n","Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from transformers==2.8.0) (1.19.5)\n","Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers==2.8.0) (2.23.0)\n","Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers==2.8.0) (4.41.1)\n","Requirement already satisfied: tokenizers==0.5.2 in /usr/local/lib/python3.7/dist-packages (from transformers==2.8.0) (0.5.2)\n","Requirement already satisfied: botocore<1.21.0,>=1.20.104 in /usr/local/lib/python3.7/dist-packages (from boto3->transformers==2.8.0) (1.20.104)\n","Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.7/dist-packages (from boto3->transformers==2.8.0) (0.10.0)\n","Requirement already satisfied: s3transfer<0.5.0,>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from boto3->transformers==2.8.0) (0.4.2)\n","Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==2.8.0) (1.0.1)\n","Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==2.8.0) (1.15.0)\n","Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==2.8.0) (7.1.2)\n","Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==2.8.0) (1.24.3)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==2.8.0) (2021.5.30)\n","Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==2.8.0) (3.0.4)\n","Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==2.8.0) (2.10)\n","Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.7/dist-packages (from botocore<1.21.0,>=1.20.104->boto3->transformers==2.8.0) (2.8.1)\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"HZFv4UU8sLTM","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625202707833,"user_tz":-330,"elapsed":7467,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"85a15268-82d6-4f26-8f32-bafe32f72946"},"source":["!pip install git+https://github.com/geoopt/geoopt.git\n","! pip install git+https://github.com/ferrine/hyrnn.git"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Collecting git+https://github.com/geoopt/geoopt.git\n","  Cloning https://github.com/geoopt/geoopt.git to /tmp/pip-req-build-rflxuxbt\n","  Running command git clone -q https://github.com/geoopt/geoopt.git /tmp/pip-req-build-rflxuxbt\n","Requirement already satisfied (use --upgrade to upgrade): geoopt==0.4.0rc1 from git+https://github.com/geoopt/geoopt.git in /usr/local/lib/python3.7/dist-packages\n","Requirement already satisfied: torch>=1.9.0 in /usr/local/lib/python3.7/dist-packages (from geoopt==0.4.0rc1) (1.9.0+cu102)\n","Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from geoopt==0.4.0rc1) (1.19.5)\n","Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch>=1.9.0->geoopt==0.4.0rc1) (3.7.4.3)\n","Building wheels for collected packages: geoopt\n","  Building wheel for geoopt (setup.py) ... \u001b[?25l\u001b[?25hdone\n","  Created wheel for geoopt: filename=geoopt-0.4.0rc1-cp37-none-any.whl size=85509 sha256=84d2d923bad55c58d4084875a1f2cb82c3a7f63311feb5731e1fad8283f28ee3\n","  Stored in directory: /tmp/pip-ephem-wheel-cache-b3ukri9j/wheels/10/df/30/e0d857f034c142ca5f38af048b62aae3da773b272553e5dd21\n","Successfully built geoopt\n","Collecting git+https://github.com/ferrine/hyrnn.git\n","  Cloning https://github.com/ferrine/hyrnn.git to /tmp/pip-req-build-ybdyb97c\n","  Running command git clone -q https://github.com/ferrine/hyrnn.git /tmp/pip-req-build-ybdyb97c\n","Requirement already satisfied (use --upgrade to upgrade): hyrnn==0.0.0 from git+https://github.com/ferrine/hyrnn.git in /usr/local/lib/python3.7/dist-packages\n","Requirement already satisfied: torch>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from hyrnn==0.0.0) (1.9.0+cu102)\n","Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from hyrnn==0.0.0) (1.19.5)\n","Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch>=1.0.0->hyrnn==0.0.0) (3.7.4.3)\n","Building wheels for collected packages: hyrnn\n","  Building wheel for hyrnn (setup.py) ... \u001b[?25l\u001b[?25hdone\n","  Created wheel for hyrnn: filename=hyrnn-0.0.0-cp37-none-any.whl size=13969 sha256=a2ed3f4813539f6fcd7d97160c2bbd983c0ec9ec7b77416b33bc5cb4f630c5d7\n","  Stored in directory: /tmp/pip-ephem-wheel-cache-xhs16xa8/wheels/24/c3/64/cc0e9d25d466081dc154a2a8843157f54d845b916b4ba66418\n","Successfully built hyrnn\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"s6vnr0t6zq8n","colab":{"base_uri":"https://localhost:8080/","height":980},"executionInfo":{"status":"ok","timestamp":1625202707834,"user_tz":-330,"elapsed":21,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"436eca72-74f9-4d97-9a75-1e41e31ae2bf"},"source":["import pandas as pd\n","train = pd.read_csv(\"train_QC_data.csv\")\n","val = pd.read_csv(\"val_QC_data.csv\")\n","test = pd.read_csv(\"test_QC_data.csv\")\n","\n","train\n"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>questionID</th>\n","      <th>originalQuestionID</th>\n","      <th>totalPossiblePoint</th>\n","      <th>AnswerKey</th>\n","      <th>isMultipleChoiceQuestion</th>\n","      <th>includesDiagram</th>\n","      <th>examName</th>\n","      <th>grade</th>\n","      <th>year</th>\n","      <th>QCLabel</th>\n","      <th>Question</th>\n","      <th>subject</th>\n","      <th>category</th>\n","      <th>fold</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>VASoL_2008_3_34</td>\n","      <td>34</td>\n","      <td>1</td>\n","      <td>C</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>Virginia Standards of Learning - Science</td>\n","      <td>3</td>\n","      <td>2008</td>\n","      <td>matter_properties of objects_TEXT</td>\n","      <td>A student is asked to bring something that fee...</td>\n","      <td>NaN</td>\n","      <td>Train</td>\n","      <td>Easy</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>MCAS_2015_8_6</td>\n","      <td>6</td>\n","      <td>1</td>\n","      <td>B</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>MCAS</td>\n","      <td>8</td>\n","      <td>2015</td>\n","      <td>celestial_FEATURES_STELLAR</td>\n","      <td>Which of the following statements best describ...</td>\n","      <td>NaN</td>\n","      <td>Test</td>\n","      <td>Easy</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>Mercury_SC_417677</td>\n","      <td>417677</td>\n","      <td>1</td>\n","      <td>B</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>Mercury</td>\n","      <td>4</td>\n","      <td>2015</td>\n","      <td>energy_LIGHT_REFLECT</td>\n","      <td>A polished metal ball looks very shiny and bri...</td>\n","      <td>NaN</td>\n","      <td>Test</td>\n","      <td>Challenge</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>Mercury_7230423</td>\n","      <td>7230423</td>\n","      <td>1</td>\n","      <td>A</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>Mercury</td>\n","      <td>9</td>\n","      <td>2015</td>\n","      <td>LIFE_EXTINCTION_MASSEX</td>\n","      <td>Which was a main force driving extensive speci...</td>\n","      <td>NaN</td>\n","      <td>Test</td>\n","      <td>Easy</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>NYSEDREGENTS_2007_8_6</td>\n","      <td>6</td>\n","      <td>1</td>\n","      <td>2</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>NYSEDREGENTS</td>\n","      <td>8</td>\n","      <td>2007</td>\n","      <td>Life_functions_features and functions_CELLBIO_...</td>\n","      <td>Compared to the amount of hereditary informati...</td>\n","      <td>NaN</td>\n","      <td>Train</td>\n","      <td>Challenge</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>5592</th>\n","      <td>Mercury_402502</td>\n","      <td>402502</td>\n","      <td>1</td>\n","      <td>D</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>Mercury</td>\n","      <td>8</td>\n","      <td>2015</td>\n","      <td>matter_chemistry_periodic table</td>\n","      <td>According to the periodic table, argon is foun...</td>\n","      <td>NaN</td>\n","      <td>Test</td>\n","      <td>Challenge</td>\n","    </tr>\n","    <tr>\n","      <th>5593</th>\n","      <td>MCAS_2006_9_20</td>\n","      <td>20</td>\n","      <td>1</td>\n","      <td>B</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>MCAS</td>\n","      <td>9</td>\n","      <td>2006</td>\n","      <td>FOR_MOMENTUM</td>\n","      <td>Which of the following has the least momentum?...</td>\n","      <td>NaN</td>\n","      <td>Train</td>\n","      <td>Challenge</td>\n","    </tr>\n","    <tr>\n","      <th>5594</th>\n","      <td>NYSEDREGENTS_2013_8_35</td>\n","      <td>35</td>\n","      <td>1</td>\n","      <td>4</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>NYSEDREGENTS</td>\n","      <td>8</td>\n","      <td>2013</td>\n","      <td>Life_functions_features and functions_PLANT_PH...</td>\n","      <td>The amount of which greenhouse gas in the air ...</td>\n","      <td>NaN</td>\n","      <td>Test</td>\n","      <td>Easy</td>\n","    </tr>\n","    <tr>\n","      <th>5595</th>\n","      <td>Mercury_7082670</td>\n","      <td>7082670</td>\n","      <td>1</td>\n","      <td>C</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>Mercury</td>\n","      <td>7</td>\n","      <td>2015</td>\n","      <td>energy_LIGHT_electromagnetic spectrum</td>\n","      <td>The visible light spectrum can be subdivided a...</td>\n","      <td>NaN</td>\n","      <td>Test</td>\n","      <td>Easy</td>\n","    </tr>\n","    <tr>\n","      <th>5596</th>\n","      <td>Mercury_7004970</td>\n","      <td>7004970</td>\n","      <td>1</td>\n","      <td>C</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>Mercury</td>\n","      <td>8</td>\n","      <td>2015</td>\n","      <td>Life_reproduction_DNA inheritance_DOMRECESS</td>\n","      <td>A scientist crosses a red-flowered plant with ...</td>\n","      <td>NaN</td>\n","      <td>Train</td>\n","      <td>Easy</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>5597 rows × 14 columns</p>\n","</div>"],"text/plain":["                  questionID originalQuestionID  ...  category       fold\n","0            VASoL_2008_3_34                 34  ...     Train       Easy\n","1              MCAS_2015_8_6                  6  ...      Test       Easy\n","2          Mercury_SC_417677             417677  ...      Test  Challenge\n","3            Mercury_7230423            7230423  ...      Test       Easy\n","4      NYSEDREGENTS_2007_8_6                  6  ...     Train  Challenge\n","...                      ...                ...  ...       ...        ...\n","5592          Mercury_402502             402502  ...      Test  Challenge\n","5593          MCAS_2006_9_20                 20  ...     Train  Challenge\n","5594  NYSEDREGENTS_2013_8_35                 35  ...      Test       Easy\n","5595         Mercury_7082670            7082670  ...      Test       Easy\n","5596         Mercury_7004970            7004970  ...     Train       Easy\n","\n","[5597 rows x 14 columns]"]},"metadata":{"tags":[]},"execution_count":7}]},{"cell_type":"code","metadata":{"id":"iQhO6qqt6lge"},"source":["from google.colab import files"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"oCGIowV49xBd","colab":{"base_uri":"https://localhost:8080/","height":1000},"executionInfo":{"status":"ok","timestamp":1625202707836,"user_tz":-330,"elapsed":16,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"56a46539-a50b-4683-e5c8-d9e4926ab1fa"},"source":["test"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>questionID</th>\n","      <th>originalQuestionID</th>\n","      <th>totalPossiblePoint</th>\n","      <th>AnswerKey</th>\n","      <th>isMultipleChoiceQuestion</th>\n","      <th>includesDiagram</th>\n","      <th>examName</th>\n","      <th>grade</th>\n","      <th>year</th>\n","      <th>QCLabel</th>\n","      <th>Question</th>\n","      <th>subject</th>\n","      <th>category</th>\n","      <th>fold</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>Mercury_409529</td>\n","      <td>409529</td>\n","      <td>1</td>\n","      <td>B</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>Mercury</td>\n","      <td>7</td>\n","      <td>2015</td>\n","      <td>science_INFERENCE_experiment design</td>\n","      <td>Robert is a fisherman who wants to find a way ...</td>\n","      <td>NaN</td>\n","      <td>Dev</td>\n","      <td>Challenge</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>Mercury_7090790</td>\n","      <td>7090790</td>\n","      <td>1</td>\n","      <td>A</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>Mercury</td>\n","      <td>7</td>\n","      <td>2015</td>\n","      <td>matter_Change of state_EVAPoration</td>\n","      <td>Which of these factors causes water to evapora...</td>\n","      <td>NaN</td>\n","      <td>Test</td>\n","      <td>Challenge</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>TIMSS_2007_8_pg7</td>\n","      <td>pg7</td>\n","      <td>1</td>\n","      <td>C</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>TIMSS</td>\n","      <td>8</td>\n","      <td>2007</td>\n","      <td>matter_chemistry_atomic</td>\n","      <td>Which statement is true about the particles of...</td>\n","      <td>NaN</td>\n","      <td>Train</td>\n","      <td>Challenge</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>Mercury_7014455</td>\n","      <td>7014455</td>\n","      <td>1</td>\n","      <td>A</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>Mercury</td>\n","      <td>8</td>\n","      <td>2015</td>\n","      <td>energy_LIGHT_GENERICPROP</td>\n","      <td>Which generates waves that are capable of trav...</td>\n","      <td>NaN</td>\n","      <td>Dev</td>\n","      <td>Easy</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>NAEP_2000_8_S21+4</td>\n","      <td>S21+4</td>\n","      <td>1</td>\n","      <td>C</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>NAEP</td>\n","      <td>8</td>\n","      <td>2000</td>\n","      <td>forces and friction</td>\n","      <td>To keep a heavy box sliding across a carpeted ...</td>\n","      <td>NaN</td>\n","      <td>Test</td>\n","      <td>Challenge</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>1395</th>\n","      <td>Mercury_SC_401827</td>\n","      <td>401827</td>\n","      <td>1</td>\n","      <td>D</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>Mercury</td>\n","      <td>5</td>\n","      <td>2015</td>\n","      <td>matter_properties of material_ELECCOND</td>\n","      <td>Metals that easily transfer electricity are ca...</td>\n","      <td>NaN</td>\n","      <td>Test</td>\n","      <td>Easy</td>\n","    </tr>\n","    <tr>\n","      <th>1396</th>\n","      <td>MDSA_2010_5_35</td>\n","      <td>35</td>\n","      <td>1</td>\n","      <td>C</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>Maryland School Assessment - Science</td>\n","      <td>5</td>\n","      <td>2010</td>\n","      <td>EARTH_human impacts_WHAT_air pollution</td>\n","      <td>Many states require vehicles to be examined an...</td>\n","      <td>NaN</td>\n","      <td>Train</td>\n","      <td>Challenge</td>\n","    </tr>\n","    <tr>\n","      <th>1397</th>\n","      <td>Mercury_7024483</td>\n","      <td>7024483</td>\n","      <td>1</td>\n","      <td>D</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>Mercury</td>\n","      <td>8</td>\n","      <td>2015</td>\n","      <td>Life_reproduction_DNA inheritance_inheritance</td>\n","      <td>Which of these is not an inherited trait in hu...</td>\n","      <td>NaN</td>\n","      <td>Train</td>\n","      <td>Challenge</td>\n","    </tr>\n","    <tr>\n","      <th>1398</th>\n","      <td>NYSEDREGENTS_2008_4_17</td>\n","      <td>17</td>\n","      <td>1</td>\n","      <td>D</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>NYSEDREGENTS</td>\n","      <td>4</td>\n","      <td>2008</td>\n","      <td>Life_functions_FUNCT_animalESS</td>\n","      <td>In order to survive, all animals need (A) heat...</td>\n","      <td>NaN</td>\n","      <td>Test</td>\n","      <td>Easy</td>\n","    </tr>\n","    <tr>\n","      <th>1399</th>\n","      <td>MCAS_2001_8_13</td>\n","      <td>13</td>\n","      <td>1</td>\n","      <td>C</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>MCAS</td>\n","      <td>8</td>\n","      <td>2001</td>\n","      <td>EARTH_INNER_PLATE_CONTDRIFT</td>\n","      <td>Scientists claim that the continents of South ...</td>\n","      <td>NaN</td>\n","      <td>Train</td>\n","      <td>Challenge</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>1400 rows × 14 columns</p>\n","</div>"],"text/plain":["                  questionID originalQuestionID  ...  category       fold\n","0             Mercury_409529             409529  ...       Dev  Challenge\n","1            Mercury_7090790            7090790  ...      Test  Challenge\n","2           TIMSS_2007_8_pg7                pg7  ...     Train  Challenge\n","3            Mercury_7014455            7014455  ...       Dev       Easy\n","4          NAEP_2000_8_S21+4              S21+4  ...      Test  Challenge\n","...                      ...                ...  ...       ...        ...\n","1395       Mercury_SC_401827             401827  ...      Test       Easy\n","1396          MDSA_2010_5_35                 35  ...     Train  Challenge\n","1397         Mercury_7024483            7024483  ...     Train  Challenge\n","1398  NYSEDREGENTS_2008_4_17                 17  ...      Test       Easy\n","1399          MCAS_2001_8_13                 13  ...     Train  Challenge\n","\n","[1400 rows x 14 columns]"]},"metadata":{"tags":[]},"execution_count":9}]},{"cell_type":"code","metadata":{"id":"DD4IAgG1XQGp"},"source":["import re\n","def clean_sentence(question):\n","  # print(question)\n","  question = re.sub('<[^>]*>', ' ',question)\n","  question = re.sub(' +', ' ', question)\n","  question = re.sub('\\xa0','',question)\n","  question = question.rstrip()\n","  question = re.sub('nan','',question)\n","  question = re.sub(u'\\u2004','',question)\n","  question = re.sub(u'\\u2009','',question)\n","\n","  # question = question.decode(\"utf-8\")\n","  # question = question.replace(u'\\u200\\d*','').encode(\"utf-8\")\n","  question = re.sub('&nbsp','',question)\n","  question = re.sub('&ndash','',question)\n","  question = re.sub('\\r','',question)\n","  question = re.sub('\\t','',question)\n","  question = re.sub('\\n',' ',question)\n","\n","  question = re.sub('MathType@.*','',question)\n","  question = re.sub('&thinsp','',question)\n","  question = re.sub('&times','',question)\n","  question = re.sub('\\u200b','',question)\n","  question = re.sub('&rarr;;;','',question)\n","\n","  return question"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"kNrGNk8f3kgh"},"source":["# final_data_1 = final_data.loc[0:71003,:]\n","# final_data_1"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"FIrS5sxE3kgk","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625202709449,"user_tz":-330,"elapsed":1625,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"a5d962c9-266d-480d-f441-3e4d5330d5ff"},"source":["from transformers import BertTokenizer\n","\n","# Load the BERT tokenizer.\n","print('Loading BERT tokenizer...')\n","tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/dtypes.py:526: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n","  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n","/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/dtypes.py:527: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n","  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n","/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/dtypes.py:528: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n","  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n","/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/dtypes.py:529: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n","  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n","/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/dtypes.py:530: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n","  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n","/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/dtypes.py:535: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n","  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"],"name":"stderr"},{"output_type":"stream","text":["Loading BERT tokenizer...\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"-mgc72PQYV1Y","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625202709450,"user_tz":-330,"elapsed":45,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"2c1087e3-3847-4fa7-f21b-42303c65247e"},"source":["train[\"QCLabel\"].value_counts()"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["celestial_cycles                                                 106\n","matter_chemistry_periodic table                                   85\n","matter_chemistry_atomic                                           79\n","matter_CHANGES_CHEMICAL                                           72\n","science_INFERENCE_observation                                     69\n","                                                                ... \n","LIFE_environment and adaptation_animal adaptations_BEHAV_HUNT      1\n","force_lift                                                         1\n","forces and friction and objects                                    1\n","LIFE_interdependency                                               1\n","celestial_FEATURES_BLACKHOLE                                       1\n","Name: QCLabel, Length: 416, dtype: int64"]},"metadata":{"tags":[]},"execution_count":13}]},{"cell_type":"code","metadata":{"id":"j3lYOb2K3kgy"},"source":["# data = [train_data,val_data,test_data]\n","# final_data = pd.concat(data)\n","# final_data"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"wp64MkNB3kg1"},"source":["# # def get_labels(prediction):\n","# #     predicted_label =  LE.inverse_transform([prediction])\n","# #     return predicted_label[0]\n","# from sklearn.model_selection import train_test_split\n","# train,val = train_test_split(final_data,test_size=.10)\n","# train,test = train_test_split(train,test_size=0.20)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"alH2aCO1nSW3","executionInfo":{"status":"ok","timestamp":1625202709451,"user_tz":-330,"elapsed":41,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"63ed471f-b963-4bde-d5d7-3e1a0b369deb"},"source":["test.shape"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["(1400, 14)"]},"metadata":{"tags":[]},"execution_count":16}]},{"cell_type":"code","metadata":{"id":"uPgTmJPS3kg4","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625202709452,"user_tz":-330,"elapsed":37,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"362e90b0-db9a-469d-9b9f-c0979efe088a"},"source":["test[\"QCLabel\"].value_counts()"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["Life_reproduction_DNA inheritance_inheritance             26\n","celestial_cycles                                          26\n","Life_functions_features and functions_PLANT_PHOTOSYNTH    22\n","matter_chemistry_atomic                                   21\n","science_INFERENCE_experiment design                       19\n","                                                          ..\n","energy_ELEC_OHMSLAW                                        1\n","matter_properties of material_LUSTER                       1\n","matter_measurement_UNIT_TIME                               1\n","energy_ELEC_ELECTROMAGNETS_energy_devices                  1\n","EARTH_human impacts_WHAT_pollution_SOIL                    1\n","Name: QCLabel, Length: 352, dtype: int64"]},"metadata":{"tags":[]},"execution_count":17}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"V4ndy386yWQa","executionInfo":{"status":"ok","timestamp":1625202709452,"user_tz":-330,"elapsed":34,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"dfd528fa-634d-4070-8a5d-d72755b17f60"},"source":["len(list(set(test[\"QCLabel\"].values).intersection(set(train[\"QCLabel\"].values))))"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["348"]},"metadata":{"tags":[]},"execution_count":18}]},{"cell_type":"code","metadata":{"id":"OVJ61SXUtY_0"},"source":["# train.to_csv(\"train_QC_data.csv\",index=False)\n","# test.to_csv(\"test_QC_data.csv\",index=False)\n","# val.to_csv(\"val_QC_data.csv\",index=False)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Oz-rOZlVx2PZ"},"source":["# train = pd.read_csv(\"train_QC_data.csv\")\n","# test = pd.read_csv(\"test_QC_data.csv\")\n","# val = pd.read_csv(\"val_QC_data.csv\")"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"HrQ1BBx0vWUQ"},"source":["# train_data = pd.concat([train_data,val_data])\n","# train_data"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"ijn1nIpByb3e"},"source":["train_features = train[\"Question\"]\n","test_features = test[\"Question\"]\n","train_labels = train[\"QCLabel\"]\n","test_labels = test[\"QCLabel\"]\n","val_features = val[\"Question\"]\n","val_labels = val[\"QCLabel\"]"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"prM_km_83khD","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625202709454,"user_tz":-330,"elapsed":30,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"87cfc2db-9fbc-492e-9940-480f090e5cc4"},"source":["train_labels.value_counts()"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["celestial_cycles                                                 106\n","matter_chemistry_periodic table                                   85\n","matter_chemistry_atomic                                           79\n","matter_CHANGES_CHEMICAL                                           72\n","science_INFERENCE_observation                                     69\n","                                                                ... \n","LIFE_environment and adaptation_animal adaptations_BEHAV_HUNT      1\n","force_lift                                                         1\n","forces and friction and objects                                    1\n","LIFE_interdependency                                               1\n","celestial_FEATURES_BLACKHOLE                                       1\n","Name: QCLabel, Length: 416, dtype: int64"]},"metadata":{"tags":[]},"execution_count":23}]},{"cell_type":"code","metadata":{"id":"qfhPstXJ03oz","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625202709455,"user_tz":-330,"elapsed":27,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"ca28d1a3-c1fe-4703-8b51-44ab2a6d5428"},"source":["test_labels.value_counts()"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["Life_reproduction_DNA inheritance_inheritance             26\n","celestial_cycles                                          26\n","Life_functions_features and functions_PLANT_PHOTOSYNTH    22\n","matter_chemistry_atomic                                   21\n","science_INFERENCE_experiment design                       19\n","                                                          ..\n","energy_ELEC_OHMSLAW                                        1\n","matter_properties of material_LUSTER                       1\n","matter_measurement_UNIT_TIME                               1\n","energy_ELEC_ELECTROMAGNETS_energy_devices                  1\n","EARTH_human impacts_WHAT_pollution_SOIL                    1\n","Name: QCLabel, Length: 352, dtype: int64"]},"metadata":{"tags":[]},"execution_count":24}]},{"cell_type":"code","metadata":{"id":"AkyM7gqv3khI"},"source":["\n","question_answer = train_features.values\n","categories = train_labels.values"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"tFkS_H_83khL","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625202709455,"user_tz":-330,"elapsed":22,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"7aa29615-659d-4c7d-8b60-8443d5175a7d"},"source":["question_answer"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["array(['A student is asked to bring something that feels rough to class. Which would be BEST for him to bring? (A) Pillow (B) Marble (C) Sandpaper (D) Trading card',\n","       'Which of the following statements best describes the role of gravity in the formation of stars? (A) Gravity converts solid matter into gases and light energy. (B) Gravity causes gases and dust particles to condense into spheres. (C) Gravity cools gases and liquids until they become one solid mass. (D) Gravity pushes rocks and dust particles outward from a dense center.',\n","       'A polished metal ball looks very shiny and bright on a sunny day. What makes the ball look shiny? (A) The ball makes light. (B) The ball reflects light. (C) The ball absorbs light and then releases it. (D) The ball absorbs light and keeps it inside.',\n","       ...,\n","       'The amount of which greenhouse gas in the air will increase the most if large forests are cut down to be used for building materials without planting new trees in their place? (1) ozone (2) methane (3) water vapor (4) carbon dioxide',\n","       'The visible light spectrum can be subdivided according to (A) the types of waves. (B) the sizes of particles. (C) a range of colors. (D) a type of energy.',\n","       'A scientist crosses a red-flowered plant with a white-flowered plant, and all offspring have red flowers. What will most likely result if these red-flowered offspring are crossed with white-flowered plants? (A) All of the offspring will have red flowers. (B) All of the offspring will have white flowers. (C) The offspring will have either red or white flowers. (D) The offspring will have neither red nor white flowers.'],\n","      dtype=object)"]},"metadata":{"tags":[]},"execution_count":26}]},{"cell_type":"code","metadata":{"id":"ian7gSDE3khR","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625202709456,"user_tz":-330,"elapsed":19,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"4e90b5af-a946-480e-dc96-9f093806ca63"},"source":["categories"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["array(['matter_properties of objects_TEXT', 'celestial_FEATURES_STELLAR',\n","       'energy_LIGHT_REFLECT', ...,\n","       'Life_functions_features and functions_PLANT_PHOTOSYNTH',\n","       'energy_LIGHT_electromagnetic spectrum',\n","       'Life_reproduction_DNA inheritance_DOMRECESS'], dtype=object)"]},"metadata":{"tags":[]},"execution_count":27}]},{"cell_type":"code","metadata":{"id":"fepGiggpqOQx"},"source":["# val_features = test_features.values\n","# val_labels = test_labels.values"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"FN1zRXMOXwLK","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625202712764,"user_tz":-330,"elapsed":3323,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"6d35ff43-637e-40eb-f4da-40559f94ccb5"},"source":["\n","import numpy as np\n","!pip install inflection\n","\n","from bokeh.io import output_file, output_notebook, show\n","from bokeh.plotting import figure\n","from bokeh.transform import linear_cmap\n","from bokeh.util.hex import hexbin\n","from bokeh.models import HoverTool\n","from bokeh import colors\n","import inflection\n","\n","from nltk.stem import PorterStemmer \n","ps = PorterStemmer()\n","from gzip import open as gopen\n","from pandas.core.common import flatten\n","import gensim.models.poincare as poincare\n","def get_cleaned_taxonomy(taxonomy):\n","  cleaned_taxonomy = []\n","  for value in taxonomy:\n","      value = ' '.join(value.lower().split(\"_\"))\n","      # taxonomy_words = [inflection.singularize(val)  for token in value for val in token.split(\" \") if val.isalpha()]\n","      cleaned_taxonomy.append( value )\n","  return cleaned_taxonomy"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Requirement already satisfied: inflection in /usr/local/lib/python3.7/dist-packages (0.5.1)\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"wPAl0TNuX6mx"},"source":["\n","# course_taxonomy\n","\n","poincare_emb_data = get_cleaned_taxonomy(categories)\n","poincare_val = get_cleaned_taxonomy(val_labels)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"aedZzkBsqEeK","colab":{"base_uri":"https://localhost:8080/","height":35},"executionInfo":{"status":"ok","timestamp":1625202712766,"user_tz":-330,"elapsed":20,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"e05d1e39-f2c8-4bba-dfd5-8be25c48348f"},"source":["poincare_emb_data[10]"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"application/vnd.google.colaboratory.intrinsic+json":{"type":"string"},"text/plain":["'life functions features and functions cellbio unimulticell'"]},"metadata":{"tags":[]},"execution_count":31}]},{"cell_type":"code","metadata":{"id":"w_ocuHxzCy16","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625202712767,"user_tz":-330,"elapsed":19,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"96294f5d-26a8-44b9-cb15-0fdbb39a577c"},"source":["import nltk\n","nltk.download('wordnet')\n","from nltk.stem import WordNetLemmatizer \n","wnl = WordNetLemmatizer()"],"execution_count":null,"outputs":[{"output_type":"stream","text":["[nltk_data] Downloading package wordnet to /root/nltk_data...\n","[nltk_data]   Package wordnet is already up-to-date!\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"BjDKIFSENir_","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625202727993,"user_tz":-330,"elapsed":15238,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"5b615447-e1ec-48fe-f3a9-b299aaeefb65"},"source":["import tensorflow_hub as hub\n","import tensorflow as tf\n","class UseSentenceEmbedding():\n","    def __init__(self):\n","        # g = tf.Graph()\n","        with tf.device('/CPU:0'):\n","        # We will be feeding 1D tensors of text into the graph.\n","            self.text_input = tf.placeholder(dtype=tf.string, shape=[None])\n","            \n","            #kindly replace the location in hub.module with the url commented out below\n","\n","            # \"https://tfhub.dev/google/universal-sentence-encoder-large/3\"\n","            embed = hub.Module(\"https://tfhub.dev/google/universal-sentence-encoder-large/3\")\n","            self.embedded_text = embed(self.text_input)\n","            init_op = tf.group([tf.global_variables_initializer(), tf.tables_initializer()])\n","        # g.finalize()\n","\n","        self.session = tf.Session(config=tf.ConfigProto( allow_soft_placement=True))\n","        self.session.run(init_op)\n","        print(\"init _____\")\n","\n","\n","\n","    def get_tokenized_sents_embeddings_USE(self, sents,expand=False):\n","\n","           \n","        vectors_USE =  self.session.run(self.embedded_text, feed_dict={self.text_input: sents})\n","\n","        return vectors_USE\n","\n","use_embedding = UseSentenceEmbedding()"],"execution_count":null,"outputs":[{"output_type":"stream","text":["INFO:absl:hub.KerasLayer is not available because TensorFlow version is less than 1.14\n","INFO:absl:Using /tmp/tfhub_modules to cache modules.\n","INFO:absl:Downloading TF-Hub Module 'https://tfhub.dev/google/universal-sentence-encoder-large/3'.\n","INFO:absl:Downloaded https://tfhub.dev/google/universal-sentence-encoder-large/3, Total size: 810.60MB\n","INFO:absl:Downloaded TF-Hub Module 'https://tfhub.dev/google/universal-sentence-encoder-large/3'.\n"],"name":"stderr"},{"output_type":"stream","text":["init _____\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"LdBbsrO9zp2p","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625202774926,"user_tz":-330,"elapsed":46938,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"d22521d0-d27c-4c48-a968-03d5075ee787"},"source":["# taxonomy_vectors = []\n","taxonomy_vectors = use_embedding.get_tokenized_sents_embeddings_USE(poincare_emb_data)\n","# taxonomy_vectors = np.vstack(taxonomy_vectors)\n","taxonomy_vectors.shape"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["(5597, 512)"]},"metadata":{"tags":[]},"execution_count":34}]},{"cell_type":"code","metadata":{"id":"oTlCYX9Q34JG","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625202781660,"user_tz":-330,"elapsed":6744,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"a9ae7a8b-661c-491b-b92e-1db3bd4f56f8"},"source":["# taxonomy_vectors_val = []\n","# for feature in poincare_val:\n","taxonomy_vectors_val = use_embedding.get_tokenized_sents_embeddings_USE(poincare_val)\n","# taxonomy_vectors_val = np.vstack(taxonomy_vectors_val)\n","taxonomy_vectors_val.shape"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["(778, 512)"]},"metadata":{"tags":[]},"execution_count":35}]},{"cell_type":"code","metadata":{"id":"Y_ZeuHc63khU","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625202787357,"user_tz":-330,"elapsed":5701,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"2363853f-6fdf-42aa-d6f1-076726bebe70"},"source":["input_ids = []\n","attention_masks = []\n","\n","for sent in question_answer:\n","    # `encode_plus` will:\n","    #   (1) Tokenize the sentence.\n","    #   (2) Prepend the `[CLS]` token to the start.\n","    #   (3) Append the `[SEP]` token to the end.\n","    #   (4) Map tokens to their IDs.\n","    #   (5) Pad or truncate the sentence to `max_length`\n","    #   (6) Create attention masks for [PAD] tokens.\n","    encoded_dict = tokenizer.encode_plus(\n","                        sent,                      # Sentence to encode.\n","                        add_special_tokens = True, # Add '[CLS]' and '[SEP]'\n","                        max_length = 128,           # Pad & truncate all sentences.\n","                        pad_to_max_length = True,\n","                        truncation=True,\n","                        return_attention_mask = True,   # Construct attn. masks.\n","                        return_tensors = 'pt',     # Return pytorch tensors.\n","                   )\n","    \n","    # Add the encoded sentence to the list.    \n","    input_ids.append(encoded_dict['input_ids'])\n","    \n","    # And its attention mask (simply differentiates padding from non-padding).\n","    attention_masks.append(encoded_dict['attention_mask'])\n","\n","# Convert the lists into tensors.\n","input_ids = torch.cat(input_ids, dim=0)\n","attention_masks = torch.cat(attention_masks, dim=0)\n","\n","\n","# Print sentence 0, now as a list of IDs.\n","print('Original: ', question_answer[0])\n","print('Token IDs:', input_ids[0])"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Original:  A student is asked to bring something that feels rough to class. Which would be BEST for him to bring? (A) Pillow (B) Marble (C) Sandpaper (D) Trading card\n","Token IDs: tensor([  101,  1037,  3076,  2003,  2356,  2000,  3288,  2242,  2008,  5683,\n","         5931,  2000,  2465,  1012,  2029,  2052,  2022,  2190,  2005,  2032,\n","         2000,  3288,  1029,  1006,  1037,  1007, 10005,  1006,  1038,  1007,\n","         7720,  1006,  1039,  1007,  5472, 23298,  1006,  1040,  1007,  6202,\n","         4003,   102,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0])\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"2VjkhiN3pAfP","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625202788182,"user_tz":-330,"elapsed":829,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"0c8656f7-8499-412b-9657-4eee3cfde7ff"},"source":["input_ids_val = []\n","attention_masks_val = []\n","\n","for sent in val_features:\n","    encoded_dict = tokenizer.encode_plus(\n","                        sent,                      # Sentence to encode.\n","                        add_special_tokens = True, # Add '[CLS]' and '[SEP]'\n","                        max_length = 128,           # Pad & truncate all sentences.\n","                        pad_to_max_length = True,\n","                        truncation=True,\n","                        return_attention_mask = True,   # Construct attn. masks.\n","                        return_tensors = 'pt',     # Return pytorch tensors.\n","                   )\n","    \n","    # Add the encoded sentence to the list.    \n","    input_ids_val.append(encoded_dict['input_ids'])\n","    \n","    # And its attention mask (simply differentiates padding from non-padding).\n","    attention_masks_val.append(encoded_dict['attention_mask'])\n","\n","# Convert the lists into tensors.\n","input_ids_val = torch.cat(input_ids_val, dim=0)\n","attention_masks_val = torch.cat(attention_masks_val, dim=0)\n","\n","\n","# Print sentence 0, now as a list of IDs.\n","print('Original: ', question_answer[0])\n","print('Token IDs:', input_ids[0])"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Original:  A student is asked to bring something that feels rough to class. Which would be BEST for him to bring? (A) Pillow (B) Marble (C) Sandpaper (D) Trading card\n","Token IDs: tensor([  101,  1037,  3076,  2003,  2356,  2000,  3288,  2242,  2008,  5683,\n","         5931,  2000,  2465,  1012,  2029,  2052,  2022,  2190,  2005,  2032,\n","         2000,  3288,  1029,  1006,  1037,  1007, 10005,  1006,  1038,  1007,\n","         7720,  1006,  1039,  1007,  5472, 23298,  1006,  1040,  1007,  6202,\n","         4003,   102,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0])\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"BNDW74Ny3khj","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625202788183,"user_tz":-330,"elapsed":36,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"4c358162-5cad-476f-cc9c-ee71f9dd17aa"},"source":["num_classes = len(list(set(categories)))\n","num_classes"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["416"]},"metadata":{"tags":[]},"execution_count":38}]},{"cell_type":"code","metadata":{"id":"alGj5A506f1q"},"source":["from torch.utils.data import TensorDataset, random_split\n","from torch.utils.data import DataLoader, RandomSampler, SequentialSampler\n"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"ZmaLk5Ab3khl"},"source":["from torch.utils.data import TensorDataset, random_split\n","train_poincare_tensor = torch.tensor(taxonomy_vectors,dtype=torch.float)\n","val_poincare_tensor = torch.tensor(taxonomy_vectors_val,dtype=torch.float)\n","\n","val_dataset = TensorDataset(input_ids_val,attention_masks_val,val_poincare_tensor)\n","# Combine the training inputs into a TensorDataset.\n","train_dataset = TensorDataset(input_ids, attention_masks, train_poincare_tensor)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"D_lTinod3kho"},"source":["from torch.utils.data import DataLoader, RandomSampler, SequentialSampler\n","batch_size = 32\n","train_dataloader = DataLoader(\n","            train_dataset,  # The training samples.\n","            sampler = RandomSampler(train_dataset), # Select batches randomly\n","            batch_size = batch_size # Trains with this batch size.\n","        )\n","\n","validation_dataloader = DataLoader(\n","            val_dataset, # The validation samples.\n","            sampler = SequentialSampler(val_dataset), \n","            batch_size = batch_size \n","        )"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"7Vduf9fOMviK","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625202788186,"user_tz":-330,"elapsed":26,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"62bbbb75-0bba-4871-b363-b49ba99827b7"},"source":["# !pip install transformers==2.8.0\n","import transformers\n","print(transformers.__version__)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["2.8.0\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"3H2wp8WlEi9u","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625202788186,"user_tz":-330,"elapsed":19,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"f62acd8f-5e42-4b08-dbe3-7a343552c1ed"},"source":["set(train[\"Question\"].values).intersection(set(test[\"Question\"].values))"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["set()"]},"metadata":{"tags":[]},"execution_count":43}]},{"cell_type":"code","metadata":{"id":"ZN6jdKp0uhO3"},"source":["\n","import sys\n","import json\n","import numpy as np\n","import torch\n","import torch.nn as nn\n","import torch.nn.functional as F\n","from torchvision import models\n","from sklearn.utils import shuffle\n","from sklearn.model_selection import train_test_split\n","from sklearn.metrics import f1_score\n","from sklearn.metrics import precision_recall_fscore_support\n","from matplotlib import pyplot as plt\n","from torch.nn.modules.loss import HingeEmbeddingLoss\n","from random import randint\n","\n","from tqdm import tqdm\n","import geoopt\n","import time\n","from sklearn.utils import shuffle\n","from sklearn.model_selection import train_test_split\n","from sklearn.metrics import f1_score\n","from sklearn.metrics import precision_recall_fscore_support\n","from matplotlib import pyplot as plt\n","from torch.nn.modules.loss import HingeEmbeddingLoss\n","from random import randint\n","import torch.nn.functional as F\n","\n","import time\n","import argparse\n","cos = nn.CosineSimilarity(dim=0, eps=1e-6)\n","# Neural Classifierwork\n","class MulticlassClassifier(nn.Module):\n","    def __init__(self,bert_model_path):\n","        super(MulticlassClassifier,self).__init__()\n","        self.bert = BertModel.from_pretrained(bert_model_path,output_hidden_states=True,output_attentions=False)\n","        self.dropout = nn.Dropout(0.1)\n","        self.fc1 = nn.Linear(768, 384)\n","        self.fc2 = nn.Linear(384, 512)\n","\n","    def forward(self,tokens,masks):\n","        _, pooled_output,hidden_states = self.bert(tokens, attention_mask=masks)\n","        representation = torch.mean(hidden_states[-2],dim=1)\n","        x = self.fc1(representation)\n","        x = self.fc2(x)\n","        return x\n","\n","class MyHingeLoss(torch.nn.Module):\n","    def __init__(self, margin):\n","        super(MyHingeLoss, self).__init__()\n","        self.margin = margin\n","    # def forward_val(self, output, target):\n","    #     cos = nn.CosineSimilarity(dim=0, eps=1e-6)\n","    #     loss = 0\n","    #     num_compare = 4\n","    #     count = 0\n","    #     for i in range(len(output)):\n","    #         v_image = output[i]\n","    #         t_label = target[i]\n","    #         for j in range(num_compare):\n","    #             if j != i:\n","    #                 count += 1\n","    #                 t_j = target[j]\n","    #                 loss += torch.relu( self.margin - cos(t_label, v_image) + cos(t_j, v_image) )\n","    #     return loss / count\n","\n","    def forward(self, output, target):\n","        loss=0\n","        for i in range(len(output)):\n","            v_image = F.normalize(output[i],p=2,dim=0)\n","            t_label = F.normalize(target[i],p=2,dim=0)\n","            j = randint(0, len(output)-1)\n","            while j == i:\n","                j = randint(0, len(output)-1)\n","            t_j = F.normalize(target[j],p=2,dim=0)\n","            loss+= torch.relu( self.margin - cos(t_label, v_image) + cos(t_j, v_image) )\n","        return loss / len(output)\n"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"yHYyjxMDIx2U"},"source":["from transformers import BertModel, AdamW, BertConfig\n","from torch.utils.data import DataLoader, RandomSampler, SequentialSampler\n"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"NiD7ymaegh3Z"},"source":["# Training Loop  \n","Do not execute this section if testing for only inference"]},{"cell_type":"code","metadata":{"id":"_2tmAMlw3khr","colab":{"base_uri":"https://localhost:8080/"},"outputId":"2d9f1378-53dd-475a-c849-81a9dee7644c"},"source":["from transformers import BertModel, AdamW, BertConfig\n","\n","# Loads BertModel, the pretrained BERT model with a single \n","model = MulticlassClassifier('bert-base-uncased')\n","# model.load_state_dict(torch.load('model_euclidean_USE_cos_final/model_weights'))\n","\n","# Tell pytorch to run this model on the GPU.\n","model.cuda()\n"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["MulticlassClassifier(\n","  (bert): BertModel(\n","    (embeddings): BertEmbeddings(\n","      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n","      (position_embeddings): Embedding(512, 768)\n","      (token_type_embeddings): Embedding(2, 768)\n","      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","      (dropout): Dropout(p=0.1, inplace=False)\n","    )\n","    (encoder): BertEncoder(\n","      (layer): ModuleList(\n","        (0): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=768, out_features=768, bias=True)\n","              (key): Linear(in_features=768, out_features=768, bias=True)\n","              (value): Linear(in_features=768, out_features=768, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=768, out_features=768, bias=True)\n","              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=768, out_features=3072, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=3072, out_features=768, bias=True)\n","            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (1): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=768, out_features=768, bias=True)\n","              (key): Linear(in_features=768, out_features=768, bias=True)\n","              (value): Linear(in_features=768, out_features=768, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=768, out_features=768, bias=True)\n","              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=768, out_features=3072, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=3072, out_features=768, bias=True)\n","            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (2): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=768, out_features=768, bias=True)\n","              (key): Linear(in_features=768, out_features=768, bias=True)\n","              (value): Linear(in_features=768, out_features=768, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=768, out_features=768, bias=True)\n","              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=768, out_features=3072, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=3072, out_features=768, bias=True)\n","            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (3): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=768, out_features=768, bias=True)\n","              (key): Linear(in_features=768, out_features=768, bias=True)\n","              (value): Linear(in_features=768, out_features=768, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=768, out_features=768, bias=True)\n","              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=768, out_features=3072, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=3072, out_features=768, bias=True)\n","            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (4): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=768, out_features=768, bias=True)\n","              (key): Linear(in_features=768, out_features=768, bias=True)\n","              (value): Linear(in_features=768, out_features=768, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=768, out_features=768, bias=True)\n","              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=768, out_features=3072, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=3072, out_features=768, bias=True)\n","            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (5): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=768, out_features=768, bias=True)\n","              (key): Linear(in_features=768, out_features=768, bias=True)\n","              (value): Linear(in_features=768, out_features=768, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=768, out_features=768, bias=True)\n","              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=768, out_features=3072, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=3072, out_features=768, bias=True)\n","            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (6): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=768, out_features=768, bias=True)\n","              (key): Linear(in_features=768, out_features=768, bias=True)\n","              (value): Linear(in_features=768, out_features=768, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=768, out_features=768, bias=True)\n","              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=768, out_features=3072, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=3072, out_features=768, bias=True)\n","            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (7): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=768, out_features=768, bias=True)\n","              (key): Linear(in_features=768, out_features=768, bias=True)\n","              (value): Linear(in_features=768, out_features=768, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=768, out_features=768, bias=True)\n","              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=768, out_features=3072, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=3072, out_features=768, bias=True)\n","            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (8): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=768, out_features=768, bias=True)\n","              (key): Linear(in_features=768, out_features=768, bias=True)\n","              (value): Linear(in_features=768, out_features=768, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=768, out_features=768, bias=True)\n","              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=768, out_features=3072, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=3072, out_features=768, bias=True)\n","            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (9): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=768, out_features=768, bias=True)\n","              (key): Linear(in_features=768, out_features=768, bias=True)\n","              (value): Linear(in_features=768, out_features=768, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=768, out_features=768, bias=True)\n","              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=768, out_features=3072, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=3072, out_features=768, bias=True)\n","            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (10): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=768, out_features=768, bias=True)\n","              (key): Linear(in_features=768, out_features=768, bias=True)\n","              (value): Linear(in_features=768, out_features=768, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=768, out_features=768, bias=True)\n","              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=768, out_features=3072, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=3072, out_features=768, bias=True)\n","            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (11): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=768, out_features=768, bias=True)\n","              (key): Linear(in_features=768, out_features=768, bias=True)\n","              (value): Linear(in_features=768, out_features=768, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=768, out_features=768, bias=True)\n","              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=768, out_features=3072, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=3072, out_features=768, bias=True)\n","            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","      )\n","    )\n","    (pooler): BertPooler(\n","      (dense): Linear(in_features=768, out_features=768, bias=True)\n","      (activation): Tanh()\n","    )\n","  )\n","  (dropout): Dropout(p=0.1, inplace=False)\n","  (fc1): Linear(in_features=768, out_features=384, bias=True)\n","  (fc2): Linear(in_features=384, out_features=512, bias=True)\n",")"]},"metadata":{"tags":[]},"execution_count":54}]},{"cell_type":"code","metadata":{"id":"x4paz_8iTZ9o"},"source":["# mobius_params = []\n","# bert_params = []\n","\n","# def mobius_params():\n","#   for param in model.named_parameters():\n","#     if 'fc' in param[0]:\n","#       yield param[1]\n","# def bert_params():\n","#   for param in model.named_parameters():\n","#     if 'bert' in param[0]:\n","#       yield param[1]\n"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"awQ2Y9Jb3kht"},"source":["optimizer_1 = torch.optim.AdamW(model.parameters(),\n","                  lr = 2e-5, # args.learning_rate - default is 5e-5, our notebook had 2e-5\n","                  eps = 1e-8 # args.adam_epsilon  - default is 1e-8.\n","                )\n","# optimizer_2 = radam_.RiemannianAdam(mobius_params(), lr=0.01, stabilize=10)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"2Ys-M4-e3khv"},"source":["from transformers import get_linear_schedule_with_warmup\n","\n","\n","epochs = 30\n","\n","# Total number of training steps is [number of batches] x [number of epochs]. \n","total_steps = len(train_dataloader) * epochs\n","\n"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"QrYqErOD3khx","colab":{"base_uri":"https://localhost:8080/"},"outputId":"f618af4b-864d-403a-bc3f-5d702dd91ec3"},"source":["len(train_dataloader) "],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["175"]},"metadata":{"tags":[]},"execution_count":58}]},{"cell_type":"code","metadata":{"id":"EWVSE9LM3kh0","colab":{"base_uri":"https://localhost:8080/"},"outputId":"20fcd6eb-f50a-4046-e904-9b5672fc2575"},"source":["1935 * 32"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["61920"]},"metadata":{"tags":[]},"execution_count":59}]},{"cell_type":"code","metadata":{"id":"rcvxVVi63kh3"},"source":["scheduler = get_linear_schedule_with_warmup(optimizer_1, \n","                                            num_warmup_steps = 0, # Default value in run_glue.py\n","                                            num_training_steps = total_steps)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"GUw3zm6g3kh5"},"source":["# import numpy as np\n","\n","# # Function to calculate the accuracy of our predictions vs labels\n","# def flat_accuracy(preds, labels):\n","#     pred_flat = np.argmax(preds, axis=1).flatten()\n","#     labels_flat = labels.flatten()\n","#     return np.sum(pred_flat == labels_flat) / len(labels_flat)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"ta6zfUTa3kh7"},"source":["import time\n","import datetime\n","\n","def format_time(elapsed):\n","    '''\n","    Takes a time in seconds and returns a string hh:mm:ss\n","    '''\n","    # Round to the nearest second.\n","    elapsed_rounded = int(round((elapsed)))\n","    \n","    # Format as hh:mm:ss\n","    return str(datetime.timedelta(seconds=elapsed_rounded))"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"GFq9gd5kQSHb"},"source":["import os\n","os.environ['CUDA_LAUNCH_BLOCKING'] = \"1\"\n"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"KsInxVoqbsFW"},"source":["class EarlyStopping:\n","    \"\"\"Early stops the training if validation loss doesn't improve after a given patience.\"\"\"\n","    def __init__(self, patience=7, verbose=False, delta=0, path='checkpoint.pt', trace_func=print):\n","        \"\"\"\n","        Args:\n","            patience (int): How long to wait after last time validation loss improved.\n","                            Default: 7\n","            verbose (bool): If True, prints a message for each validation loss improvement. \n","                            Default: False\n","            delta (float): Minimum change in the monitored quantity to qualify as an improvement.\n","                            Default: 0\n","            path (str): Path for the checkpoint to be saved to.\n","                            Default: 'checkpoint.pt'\n","            trace_func (function): trace print function.\n","                            Default: print            \n","        \"\"\"\n","        self.patience = patience\n","        self.verbose = verbose\n","        self.counter = 0\n","        self.best_score = None\n","        self.early_stop = False\n","        self.val_loss_min = np.Inf\n","        self.delta = delta\n","        self.path = path\n","        self.trace_func = trace_func\n","    def __call__(self, val_loss, model):\n","\n","        score = -val_loss\n","\n","        if self.best_score is None:\n","            self.best_score = score\n","            self.save_checkpoint(val_loss, model)\n","        elif score < self.best_score + self.delta:\n","            self.counter += 1\n","            self.trace_func(f'EarlyStopping counter: {self.counter} out of {self.patience}')\n","            if self.counter >= self.patience:\n","                self.early_stop = True\n","        else:\n","            self.best_score = score\n","            self.save_checkpoint(val_loss, model)\n","            self.counter = 0\n","\n","    def save_checkpoint(self, val_loss, model):\n","        '''Saves model when validation loss decrease.'''\n","        if self.verbose:\n","            self.trace_func(f'Validation loss decreased ({self.val_loss_min:.6f} --> {val_loss:.6f}).  Saving model ...')\n","        torch.save(model.state_dict(), self.path)\n","        self.val_loss_min = val_loss"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Di68jXK5WaYr"},"source":["criterion = MyHingeLoss(0.1)\n"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"6LhAy2hZ3kh9","colab":{"base_uri":"https://localhost:8080/"},"outputId":"536084cf-4a5c-4d45-c9dd-051713dcbd37"},"source":["import random\n","import numpy as np\n","import json\n","from sklearn.metrics import f1_score\n","# This training code is based on the `run_glue.py` script here:\n","# https://github.com/huggingface/transformers/blob/5bfcd0485ece086ebcbed2d008813037968a9e58/examples/run_glue.py#L128\n","\n","# Set the seed value all over the place to make this reproducible.\n","seed_val = 42\n","\n","random.seed(seed_val)\n","np.random.seed(seed_val)\n","torch.manual_seed(seed_val)\n","torch.cuda.manual_seed_all(seed_val)\n","\n","# We'll store a number of quantities such as training and validation loss, \n","# validation accuracy, and timings.\n","training_stats = []\n","\n","# Measure the total training time for the whole run.\n","total_t0 = time.time()\n","early_stopping = EarlyStopping(patience=6, verbose=True)\n","\n","# For each epoch...\n","for epoch_i in range(0, epochs):\n","    \n","\n","\n","    print(\"\")\n","    print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))\n","    print('Training...')\n","\n","    # Measure how long the training epoch takes.\n","    t0 = time.time()\n","\n","    # Reset the total loss for this epoch.\n","    total_train_loss = 0\n","\n","\n","    model.train()\n","\n","    # For each batch of training data...\n","    for step, batch in enumerate(train_dataloader):\n","\n","        # Progress update every 40 batches.\n","        if step % 40 == 0 and not step == 0:\n","            # Calculate elapsed time in minutes.\n","            elapsed = format_time(time.time() - t0)\n","            \n","            # Report progress.\n","            print('  Batch {:>5,}  of  {:>5,}.    Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))\n","\n","\n","        b_input_ids = batch[0].to(device)\n","        b_input_mask = batch[1].to(device)\n","        b_labels = batch[2].to(device)\n","\n","\n","        model.zero_grad() \n","        optimizer_1.zero_grad()       \n","\n","        logits = model(b_input_ids, \n","                             b_input_mask)\n","        \n","        loss = criterion.forward(logits,b_labels)\n","\n","  \n","        total_train_loss += loss.item()\n","\n","        # Perform a backward pass to calculate the gradients.\n","        loss.backward()\n","\n","        # Clip the norm of the gradients to 1.0.\n","        # This is to help prevent the \"exploding gradients\" problem.\n","        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n","\n","        # Update parameters and take a step using the computed gradient.\n","        # The optimizer dictates the \"update rule\"--how the parameters are\n","        # modified based on their gradients, the learning rate, etc.\n","        optimizer_1.step()\n","\n","        # Update the learning rate.\n","        scheduler.step()\n","\n","    # Calculate the average loss over all of the batches.\n","    avg_train_loss = total_train_loss / len(train_dataloader)            \n","    \n","    # Measure how long this epoch took.\n","    training_time = format_time(time.time() - t0)\n","\n","    print(\"\")\n","    print(\"  Average training loss: {0:.2f}\".format(avg_train_loss))\n","    print(\"  Training epcoh took: {:}\".format(training_time))\n","        \n","    # ========================================\n","    #               Validation\n","    # ========================================\n","    # After the completion of each training epoch, measure our performance on\n","    # our validation set.\n","\n","    print(\"\")\n","    print(\"Running Validation...\")\n","\n","    t0 = time.time()\n","\n","    # Put the model in evaluation mode--the dropout layers behave differently\n","    # during evaluation.\n","    model.eval()\n","\n","    # Tracking variables \n","    total_eval_f1 = 0\n","    total_eval_loss = 0\n","    nb_eval_steps = 0\n","\n","    # Evaluate data for one epoch\n","    for batch in validation_dataloader:\n","        \n","\n","        b_input_ids = batch[0].to(device)\n","        b_input_mask = batch[1].to(device)\n","        b_labels = batch[2].to(device)\n","        \n","        # Tell pytorch not to bother with constructing the compute graph during\n","        # the forward pass, since this is only needed for backprop (training).\n","        with torch.no_grad():        \n","\n","\n","          logits = model(b_input_ids, \n","                              b_input_mask)\n","          \n","        loss = criterion(logits,b_labels)\n","\n","            \n","        # Accumulate the validation loss.\n","        total_eval_loss += loss.item()\n","        # Move logits and labels to CPU\n","        logits = logits.detach().cpu().numpy().round()\n","        label_ids = b_labels.to('cpu').numpy()\n","\n","\n","    # Calculate the average loss over all of the batches.\n","    avg_val_loss = total_eval_loss / len(validation_dataloader)\n","    early_stopping(avg_val_loss, model)\n","    if early_stopping.early_stop:\n","      print(\"Early stopping\")\n","      break  \n","    # Measure how long the validation run took.\n","    validation_time = format_time(time.time() - t0)\n","    \n","    print(\"  Validation Loss: {0:.2f}\".format(avg_val_loss))\n","    print(\"  Validation took: {:}\".format(validation_time))\n","    output_dir = 'model_euclidean_USE_cos_QC/'\n","    if not os.path.exists(output_dir):\n","        os.makedirs(output_dir)\n","\n","    print(\"Saving model to %s\" % output_dir)\n","    tokenizer.save_pretrained(output_dir)\n","    torch.save(model.state_dict(), os.path.join(output_dir, 'model_weights'))\n","\n","    !rm -rf \"/content/drive/My Drive/research_lo_content_taxonomy_classification/model_euclidean_USE_cos_QC\"\n","    !mv model_euclidean_USE_cos_QC \"/content/drive/My Drive/research_lo_content_taxonomy_classification/\"\n","    # Record all statistics from this epoch.\n","    training_stats.append(\n","        {\n","            'epoch': epoch_i + 1,\n","            'Training Loss': avg_train_loss,\n","            'Valid. Loss': avg_val_loss,\n","            'Training Time': training_time,\n","            'Validation Time': validation_time\n","        }\n","    )\n","\n","print(\"\")\n","print(\"Training complete!\")\n","\n","print(\"Total training took {:} (h:mm:ss)\".format(format_time(time.time()-total_t0)))"],"execution_count":null,"outputs":[{"output_type":"stream","text":["\n","======== Epoch 1 / 30 ========\n","Training...\n","  Batch    40  of    175.    Elapsed: 0:00:29.\n","  Batch    80  of    175.    Elapsed: 0:00:58.\n","  Batch   120  of    175.    Elapsed: 0:01:28.\n","  Batch   160  of    175.    Elapsed: 0:01:58.\n","\n","  Average training loss: 0.03\n","  Training epcoh took: 0:02:09\n","\n","Running Validation...\n","Validation loss decreased (inf --> 0.018389).  Saving model ...\n","  Validation Loss: 0.02\n","  Validation took: 0:00:08\n","Saving model to model_euclidean_USE_cos_QC/\n","\n","======== Epoch 2 / 30 ========\n","Training...\n","  Batch    40  of    175.    Elapsed: 0:00:30.\n","  Batch    80  of    175.    Elapsed: 0:01:01.\n","  Batch   120  of    175.    Elapsed: 0:01:32.\n","  Batch   160  of    175.    Elapsed: 0:02:04.\n","\n","  Average training loss: 0.01\n","  Training epcoh took: 0:02:15\n","\n","Running Validation...\n","Validation loss decreased (0.018389 --> 0.013233).  Saving model ...\n","  Validation Loss: 0.01\n","  Validation took: 0:00:08\n","Saving model to model_euclidean_USE_cos_QC/\n","\n","======== Epoch 3 / 30 ========\n","Training...\n","  Batch    40  of    175.    Elapsed: 0:00:32.\n","  Batch    80  of    175.    Elapsed: 0:01:04.\n","  Batch   120  of    175.    Elapsed: 0:01:36.\n","  Batch   160  of    175.    Elapsed: 0:02:07.\n","\n","  Average training loss: 0.01\n","  Training epcoh took: 0:02:19\n","\n","Running Validation...\n","Validation loss decreased (0.013233 --> 0.011965).  Saving model ...\n","  Validation Loss: 0.01\n","  Validation took: 0:00:08\n","Saving model to model_euclidean_USE_cos_QC/\n","\n","======== Epoch 4 / 30 ========\n","Training...\n","  Batch    40  of    175.    Elapsed: 0:00:32.\n","  Batch    80  of    175.    Elapsed: 0:01:04.\n","  Batch   120  of    175.    Elapsed: 0:01:35.\n","  Batch   160  of    175.    Elapsed: 0:02:07.\n","\n","  Average training loss: 0.01\n","  Training epcoh took: 0:02:19\n","\n","Running Validation...\n","EarlyStopping counter: 1 out of 6\n","  Validation Loss: 0.01\n","  Validation took: 0:00:07\n","Saving model to model_euclidean_USE_cos_QC/\n","\n","======== Epoch 5 / 30 ========\n","Training...\n","  Batch    40  of    175.    Elapsed: 0:00:32.\n","  Batch    80  of    175.    Elapsed: 0:01:04.\n","  Batch   120  of    175.    Elapsed: 0:01:35.\n","  Batch   160  of    175.    Elapsed: 0:02:07.\n","\n","  Average training loss: 0.01\n","  Training epcoh took: 0:02:19\n","\n","Running Validation...\n","Validation loss decreased (0.011965 --> 0.008330).  Saving model ...\n","  Validation Loss: 0.01\n","  Validation took: 0:00:08\n","Saving model to model_euclidean_USE_cos_QC/\n","\n","======== Epoch 6 / 30 ========\n","Training...\n","  Batch    40  of    175.    Elapsed: 0:00:32.\n","  Batch    80  of    175.    Elapsed: 0:01:04.\n","  Batch   120  of    175.    Elapsed: 0:01:35.\n","  Batch   160  of    175.    Elapsed: 0:02:07.\n","\n","  Average training loss: 0.01\n","  Training epcoh took: 0:02:19\n","\n","Running Validation...\n","EarlyStopping counter: 1 out of 6\n","  Validation Loss: 0.01\n","  Validation took: 0:00:07\n","Saving model to model_euclidean_USE_cos_QC/\n","\n","======== Epoch 7 / 30 ========\n","Training...\n","  Batch    40  of    175.    Elapsed: 0:00:32.\n","  Batch    80  of    175.    Elapsed: 0:01:03.\n","  Batch   120  of    175.    Elapsed: 0:01:35.\n","  Batch   160  of    175.    Elapsed: 0:02:06.\n","\n","  Average training loss: 0.01\n","  Training epcoh took: 0:02:18\n","\n","Running Validation...\n","EarlyStopping counter: 2 out of 6\n","  Validation Loss: 0.01\n","  Validation took: 0:00:07\n","Saving model to model_euclidean_USE_cos_QC/\n","\n","======== Epoch 8 / 30 ========\n","Training...\n","  Batch    40  of    175.    Elapsed: 0:00:32.\n","  Batch    80  of    175.    Elapsed: 0:01:03.\n","  Batch   120  of    175.    Elapsed: 0:01:35.\n","  Batch   160  of    175.    Elapsed: 0:02:06.\n","\n","  Average training loss: 0.01\n","  Training epcoh took: 0:02:18\n","\n","Running Validation...\n","EarlyStopping counter: 3 out of 6\n","  Validation Loss: 0.01\n","  Validation took: 0:00:07\n","Saving model to model_euclidean_USE_cos_QC/\n","\n","======== Epoch 9 / 30 ========\n","Training...\n","  Batch    40  of    175.    Elapsed: 0:00:32.\n","  Batch    80  of    175.    Elapsed: 0:01:03.\n","  Batch   120  of    175.    Elapsed: 0:01:35.\n","  Batch   160  of    175.    Elapsed: 0:02:06.\n","\n","  Average training loss: 0.01\n","  Training epcoh took: 0:02:18\n","\n","Running Validation...\n","Validation loss decreased (0.008330 --> 0.007315).  Saving model ...\n","  Validation Loss: 0.01\n","  Validation took: 0:00:08\n","Saving model to model_euclidean_USE_cos_QC/\n","\n","======== Epoch 10 / 30 ========\n","Training...\n","  Batch    40  of    175.    Elapsed: 0:00:32.\n","  Batch    80  of    175.    Elapsed: 0:01:03.\n","  Batch   120  of    175.    Elapsed: 0:01:35.\n","  Batch   160  of    175.    Elapsed: 0:02:06.\n","\n","  Average training loss: 0.00\n","  Training epcoh took: 0:02:18\n","\n","Running Validation...\n","EarlyStopping counter: 1 out of 6\n","  Validation Loss: 0.01\n","  Validation took: 0:00:07\n","Saving model to model_euclidean_USE_cos_QC/\n","\n","======== Epoch 11 / 30 ========\n","Training...\n","  Batch    40  of    175.    Elapsed: 0:00:32.\n","  Batch    80  of    175.    Elapsed: 0:01:04.\n","  Batch   120  of    175.    Elapsed: 0:01:35.\n","  Batch   160  of    175.    Elapsed: 0:02:06.\n","\n","  Average training loss: 0.00\n","  Training epcoh took: 0:02:18\n","\n","Running Validation...\n","EarlyStopping counter: 2 out of 6\n","  Validation Loss: 0.01\n","  Validation took: 0:00:07\n","Saving model to model_euclidean_USE_cos_QC/\n","\n","======== Epoch 12 / 30 ========\n","Training...\n","  Batch    40  of    175.    Elapsed: 0:00:32.\n","  Batch    80  of    175.    Elapsed: 0:01:03.\n","  Batch   120  of    175.    Elapsed: 0:01:35.\n","  Batch   160  of    175.    Elapsed: 0:02:06.\n","\n","  Average training loss: 0.00\n","  Training epcoh took: 0:02:18\n","\n","Running Validation...\n","Validation loss decreased (0.007315 --> 0.007315).  Saving model ...\n","  Validation Loss: 0.01\n","  Validation took: 0:00:08\n","Saving model to model_euclidean_USE_cos_QC/\n","\n","======== Epoch 13 / 30 ========\n","Training...\n","  Batch    40  of    175.    Elapsed: 0:00:32.\n","  Batch    80  of    175.    Elapsed: 0:01:04.\n","  Batch   120  of    175.    Elapsed: 0:01:35.\n","  Batch   160  of    175.    Elapsed: 0:02:06.\n","\n","  Average training loss: 0.00\n","  Training epcoh took: 0:02:18\n","\n","Running Validation...\n","EarlyStopping counter: 1 out of 6\n","  Validation Loss: 0.01\n","  Validation took: 0:00:07\n","Saving model to model_euclidean_USE_cos_QC/\n","\n","======== Epoch 14 / 30 ========\n","Training...\n","  Batch    40  of    175.    Elapsed: 0:00:31.\n","  Batch    80  of    175.    Elapsed: 0:01:03.\n","  Batch   120  of    175.    Elapsed: 0:01:34.\n","  Batch   160  of    175.    Elapsed: 0:02:06.\n","\n","  Average training loss: 0.00\n","  Training epcoh took: 0:02:17\n","\n","Running Validation...\n","EarlyStopping counter: 2 out of 6\n","  Validation Loss: 0.01\n","  Validation took: 0:00:07\n","Saving model to model_euclidean_USE_cos_QC/\n","\n","======== Epoch 15 / 30 ========\n","Training...\n","  Batch    40  of    175.    Elapsed: 0:00:32.\n","  Batch    80  of    175.    Elapsed: 0:01:03.\n","  Batch   120  of    175.    Elapsed: 0:01:34.\n","  Batch   160  of    175.    Elapsed: 0:02:06.\n","\n","  Average training loss: 0.00\n","  Training epcoh took: 0:02:17\n","\n","Running Validation...\n","EarlyStopping counter: 3 out of 6\n","  Validation Loss: 0.01\n","  Validation took: 0:00:07\n","Saving model to model_euclidean_USE_cos_QC/\n","\n","======== Epoch 16 / 30 ========\n","Training...\n","  Batch    40  of    175.    Elapsed: 0:00:32.\n","  Batch    80  of    175.    Elapsed: 0:01:03.\n","  Batch   120  of    175.    Elapsed: 0:01:35.\n","  Batch   160  of    175.    Elapsed: 0:02:06.\n","\n","  Average training loss: 0.00\n","  Training epcoh took: 0:02:18\n","\n","Running Validation...\n","Validation loss decreased (0.007315 --> 0.007031).  Saving model ...\n","  Validation Loss: 0.01\n","  Validation took: 0:00:08\n","Saving model to model_euclidean_USE_cos_QC/\n","\n","======== Epoch 17 / 30 ========\n","Training...\n","  Batch    40  of    175.    Elapsed: 0:00:32.\n","  Batch    80  of    175.    Elapsed: 0:01:03.\n","  Batch   120  of    175.    Elapsed: 0:01:34.\n","  Batch   160  of    175.    Elapsed: 0:02:06.\n","\n","  Average training loss: 0.00\n","  Training epcoh took: 0:02:17\n","\n","Running Validation...\n","EarlyStopping counter: 1 out of 6\n","  Validation Loss: 0.01\n","  Validation took: 0:00:07\n","Saving model to model_euclidean_USE_cos_QC/\n","\n","======== Epoch 18 / 30 ========\n","Training...\n","  Batch    40  of    175.    Elapsed: 0:00:32.\n","  Batch    80  of    175.    Elapsed: 0:01:03.\n","  Batch   120  of    175.    Elapsed: 0:01:34.\n","  Batch   160  of    175.    Elapsed: 0:02:06.\n","\n","  Average training loss: 0.00\n","  Training epcoh took: 0:02:18\n","\n","Running Validation...\n","EarlyStopping counter: 2 out of 6\n","  Validation Loss: 0.01\n","  Validation took: 0:00:07\n","Saving model to model_euclidean_USE_cos_QC/\n","\n","======== Epoch 19 / 30 ========\n","Training...\n","  Batch    40  of    175.    Elapsed: 0:00:32.\n","  Batch    80  of    175.    Elapsed: 0:01:03.\n","  Batch   120  of    175.    Elapsed: 0:01:34.\n","  Batch   160  of    175.    Elapsed: 0:02:05.\n","\n","  Average training loss: 0.00\n","  Training epcoh took: 0:02:17\n","\n","Running Validation...\n","EarlyStopping counter: 3 out of 6\n","  Validation Loss: 0.01\n","  Validation took: 0:00:07\n","Saving model to model_euclidean_USE_cos_QC/\n","\n","======== Epoch 20 / 30 ========\n","Training...\n","  Batch    40  of    175.    Elapsed: 0:00:32.\n","  Batch    80  of    175.    Elapsed: 0:01:03.\n","  Batch   120  of    175.    Elapsed: 0:01:34.\n","  Batch   160  of    175.    Elapsed: 0:02:06.\n","\n","  Average training loss: 0.00\n","  Training epcoh took: 0:02:17\n","\n","Running Validation...\n","EarlyStopping counter: 4 out of 6\n","  Validation Loss: 0.01\n","  Validation took: 0:00:07\n","Saving model to model_euclidean_USE_cos_QC/\n","\n","======== Epoch 21 / 30 ========\n","Training...\n","  Batch    40  of    175.    Elapsed: 0:00:32.\n","  Batch    80  of    175.    Elapsed: 0:01:03.\n","  Batch   120  of    175.    Elapsed: 0:01:34.\n","  Batch   160  of    175.    Elapsed: 0:02:06.\n","\n","  Average training loss: 0.00\n","  Training epcoh took: 0:02:18\n","\n","Running Validation...\n","EarlyStopping counter: 5 out of 6\n","  Validation Loss: 0.01\n","  Validation took: 0:00:07\n","Saving model to model_euclidean_USE_cos_QC/\n","\n","======== Epoch 22 / 30 ========\n","Training...\n","  Batch    40  of    175.    Elapsed: 0:00:32.\n","  Batch    80  of    175.    Elapsed: 0:01:04.\n","  Batch   120  of    175.    Elapsed: 0:01:35.\n","  Batch   160  of    175.    Elapsed: 0:02:06.\n","\n","  Average training loss: 0.00\n","  Training epcoh took: 0:02:18\n","\n","Running Validation...\n","EarlyStopping counter: 6 out of 6\n","Early stopping\n","\n","Training complete!\n","Total training took 0:54:13 (h:mm:ss)\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"6RACcsko3kh_","colab":{"base_uri":"https://localhost:8080/","height":698},"outputId":"c977618f-e817-487d-8be1-09c147ff8f07"},"source":["import pandas as pd\n","\n","# Display floats with two decimal places.\n","pd.set_option('precision', 2)\n","\n","# Create a DataFrame from our training statistics.\n","df_stats = pd.DataFrame(data=training_stats)\n","\n","# Use the 'epoch' as the row index.\n","df_stats = df_stats.set_index('epoch')\n","\n","# A hack to force the column headers to wrap.\n","#df = df.style.set_table_styles([dict(selector=\"th\",props=[('max-width', '70px')])])\n","\n","# Display the table.\n","df_stats"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>Training Loss</th>\n","      <th>Valid. Loss</th>\n","      <th>Training Time</th>\n","      <th>Validation Time</th>\n","    </tr>\n","    <tr>\n","      <th>epoch</th>\n","      <th></th>\n","      <th></th>\n","      <th></th>\n","      <th></th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>1</th>\n","      <td>3.22e-02</td>\n","      <td>1.84e-02</td>\n","      <td>0:02:09</td>\n","      <td>0:00:08</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>1.32e-02</td>\n","      <td>1.32e-02</td>\n","      <td>0:02:15</td>\n","      <td>0:00:08</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>9.99e-03</td>\n","      <td>1.20e-02</td>\n","      <td>0:02:19</td>\n","      <td>0:00:08</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>9.09e-03</td>\n","      <td>1.24e-02</td>\n","      <td>0:02:19</td>\n","      <td>0:00:07</td>\n","    </tr>\n","    <tr>\n","      <th>5</th>\n","      <td>6.99e-03</td>\n","      <td>8.33e-03</td>\n","      <td>0:02:19</td>\n","      <td>0:00:08</td>\n","    </tr>\n","    <tr>\n","      <th>6</th>\n","      <td>6.60e-03</td>\n","      <td>1.07e-02</td>\n","      <td>0:02:19</td>\n","      <td>0:00:07</td>\n","    </tr>\n","    <tr>\n","      <th>7</th>\n","      <td>5.41e-03</td>\n","      <td>9.68e-03</td>\n","      <td>0:02:18</td>\n","      <td>0:00:07</td>\n","    </tr>\n","    <tr>\n","      <th>8</th>\n","      <td>5.12e-03</td>\n","      <td>9.95e-03</td>\n","      <td>0:02:18</td>\n","      <td>0:00:07</td>\n","    </tr>\n","    <tr>\n","      <th>9</th>\n","      <td>5.19e-03</td>\n","      <td>7.32e-03</td>\n","      <td>0:02:18</td>\n","      <td>0:00:08</td>\n","    </tr>\n","    <tr>\n","      <th>10</th>\n","      <td>4.98e-03</td>\n","      <td>7.62e-03</td>\n","      <td>0:02:18</td>\n","      <td>0:00:07</td>\n","    </tr>\n","    <tr>\n","      <th>11</th>\n","      <td>4.55e-03</td>\n","      <td>7.49e-03</td>\n","      <td>0:02:18</td>\n","      <td>0:00:07</td>\n","    </tr>\n","    <tr>\n","      <th>12</th>\n","      <td>4.20e-03</td>\n","      <td>7.31e-03</td>\n","      <td>0:02:18</td>\n","      <td>0:00:08</td>\n","    </tr>\n","    <tr>\n","      <th>13</th>\n","      <td>4.02e-03</td>\n","      <td>8.36e-03</td>\n","      <td>0:02:18</td>\n","      <td>0:00:07</td>\n","    </tr>\n","    <tr>\n","      <th>14</th>\n","      <td>3.58e-03</td>\n","      <td>8.64e-03</td>\n","      <td>0:02:17</td>\n","      <td>0:00:07</td>\n","    </tr>\n","    <tr>\n","      <th>15</th>\n","      <td>3.56e-03</td>\n","      <td>8.75e-03</td>\n","      <td>0:02:17</td>\n","      <td>0:00:07</td>\n","    </tr>\n","    <tr>\n","      <th>16</th>\n","      <td>3.34e-03</td>\n","      <td>7.03e-03</td>\n","      <td>0:02:18</td>\n","      <td>0:00:08</td>\n","    </tr>\n","    <tr>\n","      <th>17</th>\n","      <td>3.18e-03</td>\n","      <td>7.33e-03</td>\n","      <td>0:02:17</td>\n","      <td>0:00:07</td>\n","    </tr>\n","    <tr>\n","      <th>18</th>\n","      <td>2.87e-03</td>\n","      <td>7.48e-03</td>\n","      <td>0:02:18</td>\n","      <td>0:00:07</td>\n","    </tr>\n","    <tr>\n","      <th>19</th>\n","      <td>3.20e-03</td>\n","      <td>7.96e-03</td>\n","      <td>0:02:17</td>\n","      <td>0:00:07</td>\n","    </tr>\n","    <tr>\n","      <th>20</th>\n","      <td>2.79e-03</td>\n","      <td>8.68e-03</td>\n","      <td>0:02:17</td>\n","      <td>0:00:07</td>\n","    </tr>\n","    <tr>\n","      <th>21</th>\n","      <td>2.55e-03</td>\n","      <td>7.35e-03</td>\n","      <td>0:02:18</td>\n","      <td>0:00:07</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["       Training Loss  Valid. Loss Training Time Validation Time\n","epoch                                                          \n","1           3.22e-02     1.84e-02       0:02:09         0:00:08\n","2           1.32e-02     1.32e-02       0:02:15         0:00:08\n","3           9.99e-03     1.20e-02       0:02:19         0:00:08\n","4           9.09e-03     1.24e-02       0:02:19         0:00:07\n","5           6.99e-03     8.33e-03       0:02:19         0:00:08\n","6           6.60e-03     1.07e-02       0:02:19         0:00:07\n","7           5.41e-03     9.68e-03       0:02:18         0:00:07\n","8           5.12e-03     9.95e-03       0:02:18         0:00:07\n","9           5.19e-03     7.32e-03       0:02:18         0:00:08\n","10          4.98e-03     7.62e-03       0:02:18         0:00:07\n","11          4.55e-03     7.49e-03       0:02:18         0:00:07\n","12          4.20e-03     7.31e-03       0:02:18         0:00:08\n","13          4.02e-03     8.36e-03       0:02:18         0:00:07\n","14          3.58e-03     8.64e-03       0:02:17         0:00:07\n","15          3.56e-03     8.75e-03       0:02:17         0:00:07\n","16          3.34e-03     7.03e-03       0:02:18         0:00:08\n","17          3.18e-03     7.33e-03       0:02:17         0:00:07\n","18          2.87e-03     7.48e-03       0:02:18         0:00:07\n","19          3.20e-03     7.96e-03       0:02:17         0:00:07\n","20          2.79e-03     8.68e-03       0:02:17         0:00:07\n","21          2.55e-03     7.35e-03       0:02:18         0:00:07"]},"metadata":{"tags":[]},"execution_count":67}]},{"cell_type":"code","metadata":{"id":"xMqasMqLUig7","colab":{"base_uri":"https://localhost:8080/","height":698},"outputId":"4347bb55-5356-44db-f33e-529be7b5bede"},"source":["import pandas as pd\n","\n","# Display floats with two decimal places.\n","pd.set_option('precision', 2)\n","\n","# Create a DataFrame from our training statistics.\n","df_stats = pd.DataFrame(data=training_stats)\n","\n","# Use the 'epoch' as the row index.\n","df_stats = df_stats.set_index('epoch')\n","\n","# A hack to force the column headers to wrap.\n","#df = df.style.set_table_styles([dict(selector=\"th\",props=[('max-width', '70px')])])\n","\n","# Display the table.\n","df_stats"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>Training Loss</th>\n","      <th>Valid. Loss</th>\n","      <th>Training Time</th>\n","      <th>Validation Time</th>\n","    </tr>\n","    <tr>\n","      <th>epoch</th>\n","      <th></th>\n","      <th></th>\n","      <th></th>\n","      <th></th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>1</th>\n","      <td>3.22e-02</td>\n","      <td>1.84e-02</td>\n","      <td>0:02:09</td>\n","      <td>0:00:08</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>1.32e-02</td>\n","      <td>1.32e-02</td>\n","      <td>0:02:15</td>\n","      <td>0:00:08</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>9.99e-03</td>\n","      <td>1.20e-02</td>\n","      <td>0:02:19</td>\n","      <td>0:00:08</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>9.09e-03</td>\n","      <td>1.24e-02</td>\n","      <td>0:02:19</td>\n","      <td>0:00:07</td>\n","    </tr>\n","    <tr>\n","      <th>5</th>\n","      <td>6.99e-03</td>\n","      <td>8.33e-03</td>\n","      <td>0:02:19</td>\n","      <td>0:00:08</td>\n","    </tr>\n","    <tr>\n","      <th>6</th>\n","      <td>6.60e-03</td>\n","      <td>1.07e-02</td>\n","      <td>0:02:19</td>\n","      <td>0:00:07</td>\n","    </tr>\n","    <tr>\n","      <th>7</th>\n","      <td>5.41e-03</td>\n","      <td>9.68e-03</td>\n","      <td>0:02:18</td>\n","      <td>0:00:07</td>\n","    </tr>\n","    <tr>\n","      <th>8</th>\n","      <td>5.12e-03</td>\n","      <td>9.95e-03</td>\n","      <td>0:02:18</td>\n","      <td>0:00:07</td>\n","    </tr>\n","    <tr>\n","      <th>9</th>\n","      <td>5.19e-03</td>\n","      <td>7.32e-03</td>\n","      <td>0:02:18</td>\n","      <td>0:00:08</td>\n","    </tr>\n","    <tr>\n","      <th>10</th>\n","      <td>4.98e-03</td>\n","      <td>7.62e-03</td>\n","      <td>0:02:18</td>\n","      <td>0:00:07</td>\n","    </tr>\n","    <tr>\n","      <th>11</th>\n","      <td>4.55e-03</td>\n","      <td>7.49e-03</td>\n","      <td>0:02:18</td>\n","      <td>0:00:07</td>\n","    </tr>\n","    <tr>\n","      <th>12</th>\n","      <td>4.20e-03</td>\n","      <td>7.31e-03</td>\n","      <td>0:02:18</td>\n","      <td>0:00:08</td>\n","    </tr>\n","    <tr>\n","      <th>13</th>\n","      <td>4.02e-03</td>\n","      <td>8.36e-03</td>\n","      <td>0:02:18</td>\n","      <td>0:00:07</td>\n","    </tr>\n","    <tr>\n","      <th>14</th>\n","      <td>3.58e-03</td>\n","      <td>8.64e-03</td>\n","      <td>0:02:17</td>\n","      <td>0:00:07</td>\n","    </tr>\n","    <tr>\n","      <th>15</th>\n","      <td>3.56e-03</td>\n","      <td>8.75e-03</td>\n","      <td>0:02:17</td>\n","      <td>0:00:07</td>\n","    </tr>\n","    <tr>\n","      <th>16</th>\n","      <td>3.34e-03</td>\n","      <td>7.03e-03</td>\n","      <td>0:02:18</td>\n","      <td>0:00:08</td>\n","    </tr>\n","    <tr>\n","      <th>17</th>\n","      <td>3.18e-03</td>\n","      <td>7.33e-03</td>\n","      <td>0:02:17</td>\n","      <td>0:00:07</td>\n","    </tr>\n","    <tr>\n","      <th>18</th>\n","      <td>2.87e-03</td>\n","      <td>7.48e-03</td>\n","      <td>0:02:18</td>\n","      <td>0:00:07</td>\n","    </tr>\n","    <tr>\n","      <th>19</th>\n","      <td>3.20e-03</td>\n","      <td>7.96e-03</td>\n","      <td>0:02:17</td>\n","      <td>0:00:07</td>\n","    </tr>\n","    <tr>\n","      <th>20</th>\n","      <td>2.79e-03</td>\n","      <td>8.68e-03</td>\n","      <td>0:02:17</td>\n","      <td>0:00:07</td>\n","    </tr>\n","    <tr>\n","      <th>21</th>\n","      <td>2.55e-03</td>\n","      <td>7.35e-03</td>\n","      <td>0:02:18</td>\n","      <td>0:00:07</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["       Training Loss  Valid. Loss Training Time Validation Time\n","epoch                                                          \n","1           3.22e-02     1.84e-02       0:02:09         0:00:08\n","2           1.32e-02     1.32e-02       0:02:15         0:00:08\n","3           9.99e-03     1.20e-02       0:02:19         0:00:08\n","4           9.09e-03     1.24e-02       0:02:19         0:00:07\n","5           6.99e-03     8.33e-03       0:02:19         0:00:08\n","6           6.60e-03     1.07e-02       0:02:19         0:00:07\n","7           5.41e-03     9.68e-03       0:02:18         0:00:07\n","8           5.12e-03     9.95e-03       0:02:18         0:00:07\n","9           5.19e-03     7.32e-03       0:02:18         0:00:08\n","10          4.98e-03     7.62e-03       0:02:18         0:00:07\n","11          4.55e-03     7.49e-03       0:02:18         0:00:07\n","12          4.20e-03     7.31e-03       0:02:18         0:00:08\n","13          4.02e-03     8.36e-03       0:02:18         0:00:07\n","14          3.58e-03     8.64e-03       0:02:17         0:00:07\n","15          3.56e-03     8.75e-03       0:02:17         0:00:07\n","16          3.34e-03     7.03e-03       0:02:18         0:00:08\n","17          3.18e-03     7.33e-03       0:02:17         0:00:07\n","18          2.87e-03     7.48e-03       0:02:18         0:00:07\n","19          3.20e-03     7.96e-03       0:02:17         0:00:07\n","20          2.79e-03     8.68e-03       0:02:17         0:00:07\n","21          2.55e-03     7.35e-03       0:02:18         0:00:07"]},"metadata":{"tags":[]},"execution_count":68}]},{"cell_type":"code","metadata":{"id":"o5TicdiP3kiC","colab":{"base_uri":"https://localhost:8080/","height":427},"outputId":"62809750-6892-4b30-aeea-4c03022e003e"},"source":["import matplotlib.pyplot as plt\n","% matplotlib inline\n","\n","import seaborn as sns\n","\n","# Use plot styling from seaborn.\n","sns.set(style='darkgrid')\n","\n","# Increase the plot size and font size.\n","sns.set(font_scale=1.5)\n","plt.rcParams[\"figure.figsize\"] = (12,6)\n","\n","# Plot the learning curve.\n","plt.plot(df_stats['Training Loss'], 'b-o', label=\"Training\")\n","plt.plot(df_stats['Valid. Loss'], 'g-o', label=\"Validation\")\n","\n","# Label the plot.\n","plt.title(\"Training & Validation Loss\")\n","plt.xlabel(\"Epoch\")\n","plt.ylabel(\"Loss\")\n","plt.legend()\n","plt.xticks([1, 2, 3, 4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20])\n","\n","plt.show()"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAvoAAAGaCAYAAAB+A+cSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeVyU1f4H8M8Mwwz74gAyoKKSgAuioiKpmSuomKaopVfUSjNzubapWfe2XO26lKal3dSiTEtBQM1dXH5pLqkpmaiJSiIICLIKzAwzvz+IyXFAGRx4YObz/ivOc875nmfk9eo7D99zHpFWq9WCiIiIiIjMiljoBRARERERkekx0SciIiIiMkNM9ImIiIiIzBATfSIiIiIiM8REn4iIiIjIDDHRJyIiIiIyQ0z0iYiMkJaWBn9/f6xatarWc8ybNw/+/v4mXJX5qu7z9vf3x7x582o0x6pVq+Dv74+0tDSTry8uLg7+/v44efKkyecmInpcEqEXQET0OIxJmBMTE9GsWbM6XE3jc+/ePXzxxRfYtWsXsrKy0KRJEwQHB2P69Onw9fWt0RyzZs3C3r17kZCQgLZt21bZR6vVon///igoKMDRo0dhY2NjytuoUydPnsSpU6cwceJEODk5Cb0cA2lpaejfvz/Gjx+Pf/3rX0Ivh4gaECb6RNSoLVmyRO/nM2fOYPPmzRg7diyCg4P1rjVp0uSx43l7eyMpKQlWVla1nuPDDz/E+++//9hrMYV33nkHO3fuREREBLp3747s7GwcPHgQ58+fr3GiHxkZib1792Lr1q145513quxz4sQJ3Lp1C2PHjjVJkp+UlASxuH7+KH3q1Cl89tlnePbZZw0S/eHDh2Po0KGwtraul7UQERmDiT4RNWrDhw/X+7m8vBybN29Gp06dDK49qKioCA4ODkbFE4lEkMlkRq/zfg0lKSwpKcGePXvQq1cvfPzxx7r2GTNmQKlU1nieXr16QaFQYMeOHXjrrbcglUoN+sTFxQGo+FJgCo/7b2AqVlZWj/Wlj4ioLrFGn4gsQr9+/TBhwgRcvHgRL774IoKDg/HMM88AqEj4ly9fjtGjRyMkJAQdOnTAwIEDsWzZMpSUlOjNU1XN+P1thw4dwqhRoxAYGIhevXph8eLFUKvVenNUVaNf2VZYWIh///vfCA0NRWBgIJ577jmcP3/e4H7u3r2L+fPnIyQkBJ07d0ZUVBQuXryICRMmoF+/fjX6TEQiEUQiUZVfPKpK1qsjFovx7LPPIi8vDwcPHjS4XlRUhH379sHPzw8dO3Y06vOuTlU1+hqNBv/73//Qr18/BAYGIiIiAtu3b69yfEpKCt577z0MHToUnTt3RlBQEEaOHImYmBi9fvPmzcNnn30GAOjfvz/8/f31/v2rq9HPzc3F+++/jz59+qBDhw7o06cP3n//fdy9e1evX+X448ePY/369RgwYAA6dOiAsLAwxMfH1+izMMalS5fw6quvIiQkBIGBgRgyZAjWrl2L8vJyvX4ZGRmYP38++vbtiw4dOiA0NBTPPfec3po0Gg2io6MxbNgwdO7cGV26dEFYWBjefvttqFQqk6+diIzHJ/pEZDHS09MxceJEhIeHY9CgQbh37x4AIDMzE7GxsRg0aBAiIiIgkUhw6tQprFu3DsnJyVi/fn2N5j9y5Ag2bdqE5557DqNGjUJiYiK++uorODs7Y9q0aTWa48UXX0STJk3w6quvIi8vD19//TWmTp2KxMRE3V8flEolJk+ejOTkZIwcORKBgYG4fPkyJk+eDGdn5xp/HjY2NhgxYgS2bt2KH3/8ERERETUe+6CRI0dizZo1iIuLQ3h4uN61nTt3orS0FKNGjQJgus/7QR999BG+/fZbdOvWDZMmTUJOTg4++OADNG/e3KDvqVOncPr0aTz99NNo1qyZ7q8b77zzDnJzc/Hyyy8DAMaOHYuioiLs378f8+fPh6urK4CH7w0pLCzE888/j9TUVIwaNQrt2rVDcnIyvv/+e5w4cQIxMTEGf0lavnw5SktLMXbsWEilUnz//feYN28eWrRoYVCCVlu//fYbJkyYAIlEgvHjx8PNzQ2HDh3CsmXLcOnSJd1fddRqNSZPnozMzEyMGzcOLVu2RFFRES5fvozTp0/j2WefBQCsWbMGK1euRN++ffHcc8/BysoKaWlpOHjwIJRKZYP5yxWRRdMSEZmRrVu3av38/LRbt27Va+/bt6/Wz89Pu2XLFoMxZWVlWqVSadC+fPlyrZ+fn/b8+fO6tps3b2r9/Py0K1euNGgLCgrS3rx5U9eu0Wi0Q4cO1fbs2VNv3rlz52r9/PyqbPv3v/+t175r1y6tn5+f9vvvv9e1fffdd1o/Pz/t6tWr9fpWtvft29fgXqpSWFionTJlirZDhw7adu3aaXfu3FmjcdWJiorStm3bVpuZmanXPmbMGG379u21OTk5Wq328T9vrVar9fPz086dO1f3c0pKitbf318bFRWlVavVuvYLFy5o/f39tX5+fnr/NsXFxQbxy8vLtf/4xz+0Xbp00VvfypUrDcZXqvx9O3HihK7tk08+0fr5+Wm/++47vb6V/z7Lly83GD98+HBtWVmZrv327dva9u3ba+fMmWMQ80GVn9H777//0H5jx47Vtm3bVpucnKxr02g02lmzZmn9/Py0P//8s1ar1WqTk5O1fn5+2i+//PKh840YMUI7ePDgR66PiITD0h0ishguLi4YOXKkQbtUKtU9fVSr1cjPz0dubi6efPJJAKiydKYq/fv31zvVRyQSISQkBNnZ2SguLq7RHJMmTdL7uUePHgCA1NRUXduhQ4dgZWWFqKgovb6jR4+Go6NjjeJoNBrMnj0bly5dwu7du/HUU0/hjTfewI4dO/T6vfvuu2jfvn2NavYjIyNRXl6OhIQEXVtKSgrOnTuHfv366TZDm+rzvl9iYiK0Wi0mT56sVzPfvn179OzZ06C/nZ2d7r/Lyspw9+5d5OXloWfPnigqKsK1a9eMXkOl/fv3o0mTJhg7dqxe+9ixY9GkSRMcOHDAYMy4ceP0yqWaNm2KVq1a4caNG7Vex/1ycnLw66+/ol+/fggICNC1i0QivPLKK7p1A9D9Dp08eRI5OTnVzung4IDMzEycPn3aJGskItNj6Q4RWYzmzZtXu3Fy48aN+OGHH3D16lVoNBq9a/n5+TWe/0EuLi4AgLy8PNjb2xs9R2WpSF5enq4tLS0NHh4eBvNJpVI0a9YMBQUFj4yTmJiIo0ePYunSpWjWrBk+/fRTzJgxA2+99RbUarWuPOPy5csIDAysUc3+oEGD4OTkhLi4OEydOhUAsHXrVgDQle1UMsXnfb+bN28CAFq3bm1wzdfXF0ePHtVrKy4uxmeffYbdu3cjIyPDYExNPsPqpKWloUOHDpBI9P8XK5FI0LJlS1y8eNFgTHW/O7du3ar1Oh5cEwA88cQTBtdat24NsVis+wy9vb0xbdo0fPnll+jVqxfatm2LHj16IDw8HB07dtSNe+211/Dqq69i/Pjx8PDwQPfu3fH0008jLCzMqD0eRFR3mOgTkcWwtbWtsv3rr7/Gf//7X/Tq1QtRUVHw8PCAtbU1MjMzMW/ePGi12hrN/7DTVx53jpqOr6nKzaPdunUDUPEl4bPPPsMrr7yC+fPnQ61WIyAgAOfPn8fChQtrNKdMJkNERAQ2bdqEs2fPIigoCNu3b4enpyd69+6t62eqz/txvP766zh8+DDGjBmDbt26wcXFBVZWVjhy5Aiio6MNvnzUtfo6KrSm5syZg8jISBw+fBinT59GbGws1q9fj5deeglvvvkmAKBz587Yv38/jh49ipMnT+LkyZP48ccfsWbNGmzatEn3JZeIhMNEn4gs3rZt2+Dt7Y21a9fqJVz/93//J+Cqquft7Y3jx4+juLhY76m+SqVCWlpajV7qVHmft27dgkKhAFCR7K9evRrTpk3Du+++C29vb/j5+WHEiBE1XltkZCQ2bdqEuLg45OfnIzs7G9OmTdP7XOvi8658In7t2jW0aNFC71pKSorezwUFBTh8+DCGDx+ODz74QO/azz//bDC3SCQyei3Xr1+HWq3We6qvVqtx48aNKp/e17XKkrKrV68aXLt27Ro0Go3Bupo3b44JEyZgwoQJKCsrw4svvoh169bhhRdegFwuBwDY29sjLCwMYWFhACr+UvPBBx8gNjYWL730Uh3fFRE9SsN6hEBEJACxWAyRSKT3JFmtVmPt2rUCrqp6/fr1Q3l5Ob799lu99i1btqCwsLBGc/Tp0wdAxWkv99ffy2QyfPLJJ3ByckJaWhrCwsIMSlAepn379mjbti127dqFjRs3QiQSGZydXxefd79+/SASifD111/rHRX5+++/GyTvlV8uHvzLQVZWlsHxmsDf9fw1LSkaMGAAcnNzDebasmULcnNzMWDAgBrNY0pyuRydO3fGoUOHcOXKFV27VqvFl19+CQAYOHAggIpTgx48HlMmk+nKoio/h9zcXIM47du31+tDRMLiE30isnjh4eH4+OOPMWXKFAwcOBBFRUX48ccfjUpw69Po0aPxww8/YMWKFfjzzz91x2vu2bMHPj4+Buf2V6Vnz56IjIxEbGwshg4diuHDh8PT0xM3b97Etm3bAFQkbZ9//jl8fX0xePDgGq8vMjISH374IX766Sd0797d4ElxXXzevr6+GD9+PL777jtMnDgRgwYNQk5ODjZu3IiAgAC9ungHBwf07NkT27dvh42NDQIDA3Hr1i1s3rwZzZo109sPAQBBQUEAgGXLlmHYsGGQyWRo06YN/Pz8qlzLSy+9hD179uCDDz7AxYsX0bZtWyQnJyM2NhatWrWqsyfdFy5cwOrVqw3aJRIJpk6digULFmDChAkYP348xo0bB3d3dxw6dAhHjx5FREQEQkNDAVSUdb377rsYNGgQWrVqBXt7e1y4cAGxsbEICgrSJfxDhgxBp06d0LFjR3h4eCA7OxtbtmyBtbU1hg4dWif3SETGaZj/FyMiqkcvvvgitFotYmNjsXDhQri7u2Pw4MEYNWoUhgwZIvTyDEilUnzzzTdYsmQJEhMTsXv3bnTs2BHR0dFYsGABSktLazTPwoUL0b17d/zwww9Yv349VCoVvL29ER4ejhdeeAFSqRRjx47Fm2++CUdHR/Tq1atG8w4bNgxLlixBWVmZwSZcoO4+7wULFsDNzQ1btmzBkiVL0LJlS/zrX/9CamqqwQbYpUuX4uOPP8bBgwcRHx+Pli1bYs6cOZBIJJg/f75e3+DgYLzxxhv44Ycf8O6770KtVmPGjBnVJvqOjo74/vvvsXLlShw8eBBxcXGQy+V47rnnMHPmTKPfxlxT58+fr/LEIqlUiqlTpyIwMBA//PADVq5cie+//x737t1D8+bN8cYbb+CFF17Q9ff398fAgQNx6tQp7NixAxqNBgqFAi+//LJevxdeeAFHjhzBhg0bUFhYCLlcjqCgILz88st6J/sQkXBE2vrY9URERHWuvLwcPXr0QMeOHWv90ikiIjIfrNEnImqEqnpq/8MPP6CgoKDKc+OJiMjysHSHiKgReuedd6BUKtG5c2dIpVL8+uuv+PHHH+Hj44MxY8YIvTwiImoAWLpDRNQIJSQkYOPGjbhx4wbu3bsHuVyOPn36YPbs2XBzcxN6eURE1AAw0SciIiIiMkOs0SciIiIiMkNM9ImIiIiIzBA349ahu3eLodHUb2WUXO6AnJyieo0pdGzes2XEtrS4QsbmPVtGbEuLK2Rs3rPlxBaCWCyCq6t9ldeY6NchjUZb74l+ZVyhCBWb92wZsS0trpCxec+WEdvS4goZm/dsObEbEpbuEBERERGZISb6RERERERmiIk+EREREZEZYqJPRERERGSGmOgTEREREZkhnrpDREREVM9KSopRVJSP8nJVncbJyhJDo9HUaYyGFFfo2KZkZWUNBwdn2NpWfXRmTTDRJyIiIqpHKpUShYV34eLiBmtrGUQiUZ3FkkjEUKvrP+kVKq7QsU1Fq9VCpSpDXt4dSCTWsLaW1moelu4QERER1aPCwjw4ODhDKrWp0ySfGi+RSASp1Ab29s4oKsqr9TxM9ImIiIjqkVqthExmK/QyqBGwsbGFSqWs9XiW7piJ47/fRtyRFOQWlKGJkwwj+/gitL2n0MsiIiKiB2g05RCLrYReBjUCYrEVNJryWo9nom8Gjv9+G9/svgTlX/VoOQVl+Gb3JQBgsk9ERNQAsWSHauJxf09YumMG4o6k6JL8Skq1BnFHUgRaEREREREJjYm+GcgpKDOqnYiIiKixmTFjKmbMmFrvYxszlu6YAbmTrMqkXu4kE2A1REREZEl69epao34xMduhUHjV8Wrofkz0zcDIPr56NfoAIJWIMbKPr4CrIiIiIkvw7rsf6P28Zcv3yMzMwMyZr+m1u7i4Plac5cs/F2RsY8ZE3wxUbriNO5KCnIIyWEvEmDg4gBtxiYiIqM6FhQ3R+/nw4UTk5+cZtD+otLQUNjY2NY5jbW1dq/U97tjGjDX6ZiK0vSeWTu+Jpzp5w9leyiSfiIiIGowZM6Zi0qRxuHjxAl555UX069cTGzd+AwD46afDePPN2Rg+PBx9+4ZizJjhiI5eh/LycoM57q+zP3v2NHr16oojRw4iOnodRowYjH79nsSMGS8jLe1mrcbOnv2KwVgA2Lp1C0aPHo5+/XpiypQonD//a6Oo++cTfTPTrKkjfjp3C2WqcsiseUYvERGRJah8n05OQRnkDfR9Onl5d/HWW3MwaFA4wsOHomnTivXt2vUjbG3tMHbseNjZ2eLMmdNYt+4LFBcX49VXZz9y3m++WQ+x2ArjxkWhsLAA33+/Ae+//w7Wrv3GJGPj42OxfPkSdOrUBWPHPo+MjAzMn/8GHB0d4e7uUfsPpB4w0TczzZs6QAvgds49+Hg6Cr0cIiIiqmMPe59O76CGs/n1zp1szJv3LiIihuu1v/fefyCT/V3CM2JEJJYuXYT4+BhMmfIKpFLpQ+dVq9X46qtvIJFUpLUuLi5Yvnwprl27itatnzBqrJOTMz79dJlurEqlwrp1a9C+fSBWrFit6/fEE22wcOF7TPSpfjX3qEjuM3KKmegTERE1Esd+y8DRpIxajU1Jz4e6XKvXplRr8PWuZPx0Ph1abTUDq9CrowI9AxW1Wsej2NjYIDx8qEH7/Un+vXvFUCpVCArqjG3b4pCaegNt2vg9dN6hQ5/RJeAA0KlTZwBAevqtRyb6D44NCuqkN/bSpYvIz8/H9OnP6vUbODAcK1d+8tC5GwIm+mbGy90eIhGQnnNP6KUQERFRPXgwyX9Uu1Dc3T30kuVK166lYO3aNTh79hcUFxfrXSsuLnrkvJUlQJUcHZ0AAIWFhY899vbtii9fzZo11+snkUigUNTNFyJTYqJvZqwlVvBwsUVGTvGjOxMREVGD0DOw9k/S31x9rNr36bwd1RXq+47fFtL9T+4rFRYWYubMqbCzc8CLL06Dt3czSKVSXLlyCWvWrIJG8+i1i8VV70nU1uBPGY8ztjHgqTtmyMvNHhl8ok9ERGQRRvbxhVSin9I1lvfp/PrrGeTn52PBgn9jzJjn0bNnb3TrFqJ7si40T8+KL18PnsSjVquRkVG7Uqv6xETfDCnk9sjMvYfyGnwLJiIiosYttL0nJg4OgNxJBqDiSX5jeZ+OWFyRit7/BF2lUiE+PkaoJekJCGgHZ2dnbN8eD7VarWvfv38PCgsLBFxZzbB0xwwp5HYo12iRdbcECrm90MshIiKiOhba3rNRJPYPCgzsCEdHJyxc+B4iI8dCJBJh795dRm0grkvW1tZ44YWpWL58Kf75z+no27c/MjIysHv3Dnh7N4NIJBJ6iQ/FJ/pmyMutIrln+Q4RERE1ZM7OLliyZDnkcjesXbsG33//Hbp2DcH06bOEXprOqFFj8c9/voHbtzPw+eef4vz5X/Hf/34CBwdHSKUyoZf3UCKtuew2aIBycoqg0dTvx+vu7og/0+7i1eX/h1F9WmNoaMt6jZ2d/egd7uYSV8jYvGfzjytkbN6zZcS2tLhCxn4w7u3bqfD09KmX2BKJWJDNuELFra/YGo0GERED0adPX8yd+06dxnrU74tYLIJc7lD1tbpaFAnHViaBq6MM6Xd48g4RERHR4ygrMzzRaM+enSgoyEfnzsECrKjmBK3RVyqV+PTTT7Ft2zYUFBQgICAAc+bMQWho6CPHZmZmYtGiRTh27Bg0Gg169OiB+fPno3nzv885zcvLw0cffYSkpCTcvn0bYrEYLVu2xIQJEzB8+HCDuqqazNlYeMnteJY+ERER0WNKSjqHNWtW4emn+8HJyRlXrlzCzp3b0bq1L/r2HSD08h5K0ER/3rx52LdvH6KiouDj44P4+HhMmTIFGzZsQOfOnasdV1xcjKioKBQXF2PatGmQSCSIjo5GVFQUEhIS4OzsDAAoKirCzZs3MXDgQCgUCmg0Gvz888+YO3cuUlNTMXv2bKPnbCwUcnv8lJQBjVYLcQPfKEJERETUUHl5ecPNzR2xsZtRUJAPJydnhIcPxbRpM2BtbS308h5KsEQ/KSkJO3fuxPz58zFp0iQAwIgRIxAREYFly5Zh48aN1Y7dtGkTUlNTERcXh3bt2gEAevfujWHDhiE6OlqXwDdr1gybNm3SGzt+/HhMmzYN33zzDWbNmqV7ql/TORsLhZs9ylTluFtQBrmz4QsqiIiIiOjRvL2bYcmS5UIvo1YEq9Hfs2cPrK2tMXr0aF2bTCZDZGQkzpw5g6ysrGrH7t27F506ddIl5ADg6+uL0NBQ7N69+5Gxvb29UVJSApVKZbI5GxovuR0A8A25RERERBZKsEQ/OTkZrVq1gr29/jnvHTt2hFarRXJycpXjNBoNLl++jA4dOhhcCwwMxI0bN1BSUqLXXlZWhtzcXKSlpSEhIQFxcXEIDg6GVCqt9ZwNXeX5+azTJyIiIrJMgpXuZGdno2nTpgbt7u7uAFDtE/28vDwolUpdvwfHarVaZGdno0WLFrr2mJgYfPjhh7qfQ0ND8d///vex5mzoHO2sYW8j4RN9IiIiIgslWKJfWlpa5QYGmazixQNVHWV0f3vl0/iqxpaWluq1DxgwAK1bt8bdu3dx+PBhZGdn6z2hr82cNVHdmaZ1zd3dEQDQwtMJdwrKdD/XZ+z6JlRcIWPzns0/rpCxec+WEdvS4goZ+/64WVliSCT1V1RRn7EaQlyhY5uaWCyu9e+tYIm+jY2NXo18pcqkuzLBflBlu1KprHasjY3+5lNPT094ela8Fnro0KF47733MHnyZOzZswc2Nja1mrMmhHphVuVLOdydZTh75U69vRykobyIxBJi857NP66QsXnPlhHb0uIKGfvBuBqNpt5eJsUXZjV+Go3mob+3DfKFWe7u7lWW52RnZwMAPDw8qhzn4uICqVSq6/fgWJFIVGUJzv3CwsKQkZGBX375xWRzNkQKuT2KSlQovGf4BYaIiIiIzJtgiX5AQACuX7+O4mL9GvLz58/rrldFLBbDz88PFy5cMLiWlJQEHx8f2NraPjR25VP6wsJCk83ZEFVuyM3ghlwiIiIiiyNYoh8eHg6VSoWYmBhdm1KpRFxcHLp06aLbqJueno6UlBS9sWFhYTh37hwuXryoa7t27RpOnDiB8PBwXVtubm6VsWNjYyESidC+fXuj52xMKo/YTOeGXCIiImpEdu3agV69uiIjI13XFhk5DAsXvvfIsT/+uN1g7OM6e/Y0evXqirNnT5tszvogWI1+UFAQwsPDsWzZMt2JNvHx8UhPT8dHH32k6zd37lycOnUKly9f1rWNGzcOMTExmDp1KiZPngwrKytER0fD3d1d9/ItANi4cSMOHDiAp59+Gt7e3sjPz8f+/ftx/vx5jBs3Dj4+PkbP2Zg0cbaB1FqMjDt8ok9ERER156235uDs2V+wY8f+aqsgXnttBn7//Tds376v2r2YQjtwYC9yc3MwZsw4oZdiEoIl+gCwZMkSrFixAtu2bUN+fj78/f3x5ZdfIjg4+KHjHBwcsGHDBixatAirV6+GRqNBSEgIFixYAFdXV12/0NBQXLp0CQkJCcjJyYG1tTX8/f2xcOFCjBo1qlZzNiZikQieTex4xCYRERHVqYEDw/Dzzz/h6NEjGDjQsBLi7t1cnDnzCwYNGlzrJH/Tpq0Qi+u2GCUxcR/++OOKQaLfqVMXJCYeq/LEyIZM0ERfJpNh7ty5mDt3brV9NmzYUGW7p6cnVq5c+dD5u3btiq5du9Z4PTWZs7Hxktvjj7Q8oZdBREREZqx376dha2uHAwf2VpnoHzx4AOXl5Rg0qPbl0FUdg15fxGJxg/0rxMMImuhT3VPI7XDiYiZKlWrYSPnPTURERKZnY2OD3r374NChAygoKECTJi561w8c2Au5XI7mzX2wbNl/cebMKWRmZsLGxgZdunTFq6/OhkLh9dAYkZHD0LlzMBYseE/Xdu1aClasWIoLF36Ds7Mzhg8fiaZNDU9u/Omnw9i+PR5XrlxGQUE+3N09MGTIMEyYUFGuDQAzZkzFuXNnAQC9elU8KPb0VCA2dgfOnj2NWbOmYeXKL9Cly98PkRMT9+G776KRmnoDdnb26NmzN155ZRZcXP6+/xkzpqKoqAj/+tcH+OSTJUhO/h2Ojk4YPfo5jB8/0bgP2kjM/Mycl1vFyTu3c++hpaeTwKshIiKiunDq9llsT9mDu2V5cJW54BnfcHT37FKvaxg4MBz79u3G4cOJGDny7xLp27czcOFCEiIjn0Ny8u+4cCEJAwaEwd3dAxkZ6UhI2IqZM1/Gd9/FGPXeopycO5g1axo0Gg3+8Y+JsLGxxfbt8bCxMXzyvmvXj7C1tcPYseNhZ2eLM2dOY926L1BcXIxXX50NAJg48QWUlJQgMzMDM2e+BgCwtbWrNv6uXTuwaNH7aN8+EK+8MgtZWZnYunUzkpN/x9q13+r9BaCgIB+vvz4Lffv2R//+g3Do0AGsWbMKrVs/gdDQnjW+Z2Mx0TdzuiM27zDRJyIiMkenbp/FpktbodJUvIj0blkeNl3aCgB4slnNS5gfV7duIXBxccWBA3v1Evut54cAACAASURBVP0DB/ZCq9Vi4MAw+Po+gb59B+iN69nzKUybNhmHDyciPHxojeNt3PgN8vPzsG7dBvj7VxzLPnhwBJ5//lmDvu+99x/IZH9/iRgxIhJLly5CfHwMpkx5BVKpFN269UBcXAzy8/MQFjbkobHVajXWrFmFJ57ww6pV/9OVFfn7B+C99xZgx454REY+p+uflZWJf//7P7qypoiI4YiMjMDOnduY6FPtebjawkos4hGbREREDdjJjDM4nvFLrcZez/8Taq1ar02lUWFjciyOZ5yCVlvzuUIV3RCiePihKNWRSCTo128AEhK24s6dbLi4yAEABw7sQ7NmzdGuXQe9/mq1GsXFRWjWrDkcHBxx5coloxL948ePITAwSJfkA4CrqyvCwgZj69YYvb73J/n37hVDqVQhKKgztm2LQ2rqDbRp42fUvV66dBF37+bqviRU6tdvID7//FP8/PMxvUTfwcEBAwaE6X62trZG27btkZ5+y6i4xmKib+YkVmJ4uNrypVlERERm6sEk/1HtdWngwHDExcXgwIF9iIx8HjduXMfVq1cwefIUAEBZWSk2bIjGrl07kJ2dBe1930KKioqMipWZeRuBgUEG7S1atDRou3YtBWvXrsHZs78YvKy1uNi4uEBFOVJFLB+9drFYjGbNmiMzM0Ov3cOjKUQikV6bo6MTUlKuGh3bGEz0LYBCbo/0O3yiT0RE1FCFKIJr/ST9nWOLcLfM8IQ9V5kLXu82HWq15nGXV2OBgUFQKLyxb98eREY+j/379wCArmRl+fKl2LVrB0aPfh4dOgTCwcEBgAjvvfe2XtJvSoWFhZg5cyrs7Bzw4ovT4O3dDFKpFFeuXMKaNaug0dT95yMWW1XZXlf3XImJvgVQyO1w7o87UJdrILES7GXIREREVAee8Q3Xq9EHAGuxNZ7xrf1Rlo9jwIBB2LDha6Sl3URi4j74+7fVPfmurMOfOXOOrn9ZWZnRT/MBoGlTT6Sl3TRo//PPG3o///rrGeTn52PhwqXo1OnvDcpVvzlXVEWbIU9PxV+xUvXm1Gq1SEu7iVatfGs0T11j1mcBvOT20Gi1yLxbIvRSiIiIyMS6e3bBuIBRcJVVHOnoKnPBuIBR9X7qTqVBgwYDAD77bDnS0m7qnZ1f1ZPtrVs3o7y83Og4oaE98dtv53H58iVd2927d7F37269fpUv2br/6blKpUJ8vH4dPwDY2trW6EtHQEA7uLo2QUJCLFSqv79gHTqUiOzsLDz5ZN1tsDUGn+hbAIVbxdFQGXeK4f3XcZtERERkPrp7dhEssX9Qq1at0aaNH44e/T+IxWL07//3JtQnn+yFvXt3wd7eAS1btsLvv/+G06dPwdnZ2eg448ZNxN69u/Daa68iMvI5yGQ22L49Hp6eCly9+oeuX2BgRzg6OmHhwvcQGTkWIpEIe/fuqnKTsr9/APbt241Vqz5BQEA72NraoVevpwz6SSQSvPLKTCxa9D5mznwZAwYMQlZWJmJjN6N1a18MG2Z48o8QmOhbAEWTv47Y5Mk7REREVA/Cwgbjjz+uoHPnYLi5uenaZ89+A2KxGPv370ZZmRKBgUFYseJzvPbaTKNjuLm5YeXK/2H58iXYsCFa74VZCxd+oOvn7OyCJUuW47PPVmDt2jVwdHTCoEGD0bVrd7z22gy9OYcPH4UrVy5h164fsXnzJnh6KqpM9AFgyJBhkEql2LjxG3z++aewt7fHwIHhmDZtZoN5i65IW9e7ACxYTk4RNJr6/Xjd3R2RnV1o0P7m6mNo08wFU59pX++x65pQcYWMzXs2/7hCxuY9W0ZsS4srZOwH496+nQpPT5+HjDAdiURcr5txhY4rdOy68KjfF7FYBLncoeprdbUoalgUcnuepU9ERERkQZjoWwiF3B63c+5Bwz/gEBEREVkEJvoWQuFmB6Vag9z8UqGXQkRERET1gIm+hfCSV2zITecbcomIiIgsAhN9C6GQ/3XEJuv0iYiIiCwCE30L4WgnhYOtNRN9IiIiIgvBRN+CeMntWLpDREREZCGY6FsQhZs9Mu4Ug69OICIiEhb/X0w18bi/J0z0LYhCbo/iUjUK76mEXgoREZHFsrKSQKVSCr0MagRUKiWsrCS1Hs9E34J4cUMuERGR4BwcXJCXlw2lsoxP9qlKWq0WSmUZ8vKy4eDgUut5av8VgRodxX1HbPq3cBV4NURERJbJ1rbi/8f5+XdQXq6u01hisRgajaZOYzSkuELHNiUrKwkcHV11vy+1wUTfgjRxkkEmtULGHT7RJyIiEpKtrf1jJXA15e7uiOzswjqP01DiCh27oWHpjgURiURQNLFj6Q4RERGRBWCib2EUcnsesUlERERkAZjoWxgvNzvcLSxDSVnd1gQSERERkbCY6FuYyg25GXyqT0RERGTWmOhbGAWP2CQiIiKyCEz0LYyHqy2sxCKkM9EnIiIiMmtM9C2MlViMpk3skHGHpTtERERE5oyJvgVSyHnEJhEREZG5Y6JvgRRye2TllUClbvxvjSMiIiKiqjHRt0BecjtotUDmXZbvEBEREZkrJvoWiEdsEhEREZk/JvoWyFNuBxGAjDus0yciIiIyV0z0LZDM2gpyZxsesUlERERkxiRCBlcqlfj000+xbds2FBQUICAgAHPmzEFoaOgjx2ZmZmLRokU4duwYNBoNevTogfnz56N58+a6PhkZGYiNjcWRI0eQmpoKsVgMPz8/TJ8+3SDGqlWr8NlnnxnEcXNzw7Fjxx7/ZhsYhdyepTtEREREZkzQRH/evHnYt28foqKi4OPjg/j4eEyZMgUbNmxA586dqx1XXFyMqKgoFBcXY9q0aZBIJIiOjkZUVBQSEhLg7OwMAEhMTMS6deswYMAAPPvss1Cr1di2bRsmTZqExYsXY8SIEQZzf/DBB7CxsdH9fP9/mxOF3A6X/rwLjUYLsVgk9HKIiIiIyMQES/STkpKwc+dOzJ8/H5MmTQIAjBgxAhEREVi2bBk2btxY7dhNmzYhNTUVcXFxaNeuHQCgd+/eGDZsGKKjozF79mwAQEhICA4dOoQmTZroxj7//PMYPnw4Vq5cWWWiP3jwYDg5OZnwThsmLzd7qNQa3CkohYeLrdDLISIiIiITE6xGf8+ePbC2tsbo0aN1bTKZDJGRkThz5gyysrKqHbt371506tRJl+QDgK+vL0JDQ7F7925dW5s2bfSSfACQSqXo06cPbt26hdLSUoO5tVotioqKoNVqH+f2GjyF3A4AN+QSERERmSvBEv3k5GS0atUK9vb2eu0dO3aEVqtFcnJyleM0Gg0uX76MDh06GFwLDAzEjRs3UFJS8tDY2dnZsLOzg0wmM7j29NNPIzg4GMHBwZg/fz7y8vKMuKvGg0dsEhEREZk3wUp3srOz0bRpU4N2d3d3AKj2iX5eXh6USqWu34NjtVotsrOz0aJFiyrHp6amYv/+/Rg6dChEor9r052cnDBhwgQEBQXB2toaJ06cwObNm3Hx4kXExMRAKpXW5jYbLAdbazjZWfPkHSIiIiIzJViiX1paCmtra4P2yqfsZWVlVY6rbK8q8a4cW1VJDgCUlJRg9uzZsLW1xZw5c/SuTZw4Ue/n8PBwtGnTBh988AESEhIwZsyYR9yRIbncwegxpuDu7lijfj4KZ9zJL61xf1PGNjWh4goZm/ds/nGFjM17tozYlhZXyNi8Z8uJ3ZAIlujb2NhApVIZtFcm8lWV1dzfrlQqqx1b1Uk55eXlmDNnDlJSUrB+/Xp4eHg8co3PP/88li5diuPHj9cq0c/JKYJGU7+1/u7ujsjOLqxRXzcnGU5ezERWVoHeXzfqI7YpCRVXyNi8Z/OPK2Rs3rNlxLa0uELG5j1bTmwhiMWiah8uC1aj7+7uXmV5TnZ2NgBUm4i7uLhAKpXq+j04ViQSVVnW88477+DIkSNYvHgxunfvXqM1isViNG3aFPn5+TXq39go5Ha4V6ZGQbHhlyYiIiIiatwES/QDAgJw/fp1FBfr14ifP39ed70qlS+9unDhgsG1pKQk+Pj4wNZW/7jIxYsXIy4uDm+//TaGDBlS4zWqVCpkZGTA1dW1xmMaE4VbxYbcdJ68Q0RERGR2BEv0w8PDoVKpEBMTo2tTKpWIi4tDly5ddBt109PTkZKSojc2LCwM586dw8WLF3Vt165dw4kTJxAeHq7Xd926dfjqq68wbdo0TJgwodr15ObmGrStX78eZWVl6N27d63usaHz+uvknXSevENERERkdgSr0Q8KCkJ4eDiWLVumOyUnPj4e6enp+Oijj3T95s6di1OnTuHy5cu6tnHjxiEmJgZTp07F5MmTYWVlhejoaLi7u+tevgUA+/fvx9KlS9GyZUu0bt0a27Zt01vDwIEDYWdXcZ583759MWTIEPj5+UEqleLkyZPYu3cvgoODERERUbcfhkBcHKSwkVohgyfvEBEREZkdwRJ9AFiyZAlWrFiBbdu2IT8/H/7+/vjyyy8RHBz80HEODg7YsGEDFi1ahNWrV0Oj0SAkJAQLFizQK7O5dOkSAODGjRt46623DOZJTEzUJfrDhg3D2bNnsWfPHqhUKnh7e2P69Ol4+eWXIZEI+jHVGZFIBIXcnmfpExEREZkhQTNYmUyGuXPnYu7cudX22bBhQ5Xtnp6eWLly5UPnnzlzJmbOnFmjtfznP/+pUT9z4yW3w4UbhmVLRERERNS4CVajTw2Dws0e+UVK3CtVC70UIiIiIjIhJvoWTiGvKF1inT4RERGReWGib+H+PnmHiT4RERGROWGib+HcXGwgsRJxQy4RERGRmWGib+GsxGI0bWKHDL40i4iIiMisMNEnHrFJREREZIaY6BO85HbIzi+BSl0u9FKIiIiIyESY6BMUcntotcDt3BKhl0JEREREJsJEn3jEJhEREZEZYqJP8GxiBxGAdG7IJSIiIjIbTPQJUmsruLnYcEMuERERkRlhok8AKk/e4RN9IiIiInPBRJ8AVLwh93ZuCTQardBLISIiIiITYKJPAACFmx3U5Rpk5/PkHSIiIiJzwESfAFQ80QeAjDus0yciIiIyB0z0CUBFjT4ApLNOn4iIiMgsMNEnAICdjQTODlJk8IhNIiIiIrPARJ90vOT2SOcRm0RERERmgYk+6SjkdsjIKYZWy5N3iIiIiBo7Jvqko5Dbo1RZjrwipdBLISIiIqLHxESfdLzkdgC4IZeIiIjIHDDRJx2FW+URm0z0iYiIiBo7Jvqk42wvha1MggxuyCUiIiJq9Jjok45IJILXXxtyiYiIiKhxY6JPehQ8YpOIiIjILDDRJz0KNzsUFCtRXKoSeilERERE9BiY6JMehbxyQy6f6hMRERE1Zkz0SQ+P2CQiIiIyD0z0SY+bsy0kVmJuyCUiIiJq5Jjokx6xWATPJnY8YpOIiIiokWOiTwa83OyQzpdmERERETVqTPTJgEJuj5z8UihV5UIvhYiIiIhqiYk+GVDI7aAFcDuX5TtEREREjRUTfTLg5VZxxCZP3iEiIiJqvJjok4GmrnYQiXiWPhEREVFjxkSfDFhLxPBwseUTfSIiIqJGTNBEX6lUYunSpejVqxc6duyIMWPG4Pjx4zUam5mZidmzZ6Nr167o0qULpk+fjps3b+r1ycjIwKpVqxAZGYlu3bohJCQEEyZMqDZGTea0FAq5PY/YJCIiImrEBE30582bh2+++QbPPPMMFixYALFYjClTpuDXX3996Lji4mJERUXhzJkzmDZtGmbNmoWLFy8iKioK+fn5un6JiYlYt24dfHx88M9//hPTp09HcXExJk2ahISEhFrNaSkUbnbIzL2Hco1G6KUQERERUS1IhAqclJSEnTt3Yv78+Zg0aRIAYMSIEYiIiMCyZcuwcePGasdu2rQJqampiIuLQ7t27QAAvXv3xrBhwxAdHY3Zs2cDAEJCQnDo0CE0adJEN/b555/H8OHDsXLlSowYMcLoOS2Fl9we5Rotsu6WQCG3F3o5RERERGQkwZ7o79mzB9bW1hg9erSuTSaTITIyEmfOnEFWVla1Y/fu3YtOnTrpEnIA8PX1RWhoKHbv3q1ra9OmjV6SDwBSqRR9+vTBrVu3UFpaavSclqIyuWf5DhEREVHjJFiin5ycjFatWsHeXv9pcceOHaHVapGcnFzlOI1Gg8uXL6NDhw4G1wIDA3Hjxg2UlJQ8NHZ2djbs7Owgk8lMNqe5UcjtAAAZ3JBLRERE1CgJluhnZ2fDw8PDoN3d3R0Aqn2in5eXB6VSqev34FitVovs7Oxq46ampmL//v0IDw+HSCQyyZzmyFYmgaujDOk8YpOIiIioURKsRr+0tBTW1tYG7ZVP2cvKyqocV9kulUqrHXt/Sc79SkpKMHv2bNja2mLOnDkmmfNh5HIHo8eYgru7o0nm8fF0QnZBqVHzmSq2sYSKK2Rs3rP5xxUyNu/ZMmJbWlwhY/OeLSd2QyJYom9jYwOVSmXQXpl0VybYD6psVyqV1Y61sbExuFZeXo45c+YgJSUF69ev1/trQm3nfJScnCJoNFqjxz0Od3dHZGcXmmQuuZMMyam5yMoq0P31o75iG0OouELG5j2bf1whY/OeLSO2pcUVMjbv2XJiC0EsFlX7cFmw0h13d/cqy3MqS2SqKusBABcXF0il0ipLabKzsyESiaoswXnnnXdw5MgRLF68GN27dzfJnObOS26HMmU57hZW/dcVIiIiImq4BEv0AwICcP36dRQX62/2PH/+vO56VcRiMfz8/HDhwgWDa0lJSfDx8YGtra1e++LFixEXF4e3334bQ4YMMcmclqDy5B2+IZeIiIio8REs0Q8PD4dKpUJMTIyuTalUIi4uDl26dEHTpk0BAOnp6UhJSdEbGxYWhnPnzuHixYu6tmvXruHEiRMIDw/X67tu3Tp89dVXmDZtGiZMmFDteoyZ01Io3P46YpMbcomIiIgaHcFq9IOCghAeHo5ly5YhOzsbLVq0QHx8PNLT0/HRRx/p+s2dOxenTp3C5cuXdW3jxo1DTEwMpk6dismTJ8PKygrR0dFwd3fXvXwLAPbv34+lS5eiZcuWaN26NbZt26a3hoEDB8LOzs6oOS2Jk5017G0kPGKTiIiIqBESLNEHgCVLlmDFihXYtm0b8vPz4e/vjy+//BLBwcEPHefg4IANGzZg0aJFWL16NTQaDUJCQrBgwQK4urrq+l26dAkAcOPGDbz11lsG8yQmJuoS/ZrOaUlEIhEUcnuk86VZRERERI2OoIm+TCbD3LlzMXfu3Gr7bNiwocp2T09PrFy58qHzz5w5EzNnzqzxemoyp6VRyO1w7uodoZdBREREREYSrEafGgeF3B6F91QoKjE8CpWIiIiIGi4m+vRQXm4VpU3pd1inT0RERNSYmCTRV6vV2Lt3L7Zs2VLlWfTUeFUesckNuURERESNi9E1+kuWLMHJkyexdetWAIBWq8XkyZNx+vRpaLVauLi4YMuWLWjRooXJF0v1T+5sA6lEjAxuyCUiIiJqVIx+ov/TTz+ha9euup8PHjyIX375BS+++CI+/vhjAMCXX35puhWSoMQiETzldnxpFhEREVEjY/QT/du3b8PHx0f386FDh9CsWTO88cYbAIA//vgDO3bsMN0KqUZO3T6L7Sl7kFeWBxeZC57xDUd3zy4mmdtLbo8/0vJMMhcRERER1Q+jn+irVCpIJH9/Pzh58iSefPJJ3c/NmzdnnX49O3X7LDZd2oq7ZXnQArhblodNl7bi1O2zJplfIbdDTkEZSpVqk8xHRERERHXP6ETf09MTv/76K4CKp/c3b95Et27ddNdzcnJ0L6Gi+rE9ZQ9UGv3jL1UaFban7DHJ/JUbcm/nsk6fiIiIqLEwunRn6NChWL16NXJzc/HHH3/AwcEBffr00V1PTk7mRtx6dres6rKa6tqNpXD76+SdO/fQ0tPJJHMSERERUd0y+on+yy+/jGeffRbnzp2DSCTC4sWL4eRUkfwVFhbi4MGDCA0NNflCqXquMhej2o3V1NUWYpGIG3KJiIiIGhGjn+hLpVIsWrSoymv29vY4evQobGxsHnthVHPP+IZj06WteuU7IojwjG+4SeaXWInh4WrLIzaJiIiIGhGjE/2HUavVcHR0NOWUVAOVp+tUnrpjI7FBiboU9tb2JouhkNvxpVlEREREjYjRpTtHjhzBqlWr9No2btyILl26oFOnTnj99dehUqmqGU11pbtnF/yn59vYPHYN/tvrX/Cwc0PslW1QaUxzUo6Xmz2y7pZAXa4xyXxEREREVLeMTvTXr1+Pa9eu6X5OSUnBokWL4OHhgSeffBK7du3Cxo0bTbpIMo5ELEFkm+HIKrmDQzd/MsmcCrkdyjVaZN0tMcl8RERERFS3jE70r127hg4dOuh+3rVrF2QyGWJjY7Fu3ToMGTIECQkJJl0kGa+93B8d3dpj941E5JXlP/Z8lUdssnyHiIiIqHEwOtHPz8+Hq6ur7ueff/4ZPXr0gIODAwCge/fuSEtLM90KqdZGtYmARqtB/NWdjz2XQl7xboR0bsglIiIiahSMTvRdXV2Rnp4OACgqKsJvv/2Grl276q6r1WqUl5ebboVUa262cgxs0QenM8/hat71x5rLRipBEycZn+gTERERNRJGJ/qdOnXCDz/8gD179mDRokUoLy/HU089pbuempoKDw8Pky6Sam+QT1+4ylyw5UoCNNrH20irkNsj4w6f6BMRERE1BkYn+rNmzYJGo8E///lPxMXFYcSIEXjiiScAAFqtFgcOHECXLl1MvlCqHamVFCPbROBWUQaO3jrxWHMp5HbIyC2GRqs10eqIiIiIqK4YfY7+E088gV27duHs2bNwdHREt27ddNcKCgowceJEhISEmHSR9Hg6uwfCz/UJ7Li2F108guAgrd35+l5yeyhVGuQWlMLN2dbEqyQiIiIiUzL6iT4AuLi4oF+/fnpJPgA4Oztj4sSJCAgIMMniyDREIhFGt3kGpeVl2HFtT63nqdyQyzfkEhERETV8tX4z7p9//onExETcvHkTANC8eXP0798fLVq0MNniyHS8HDzRp9mTOHzzGHp6h6CFYzOj51C4/XXE5p1iBLaWm3qJRERERGRCtUr0V6xYgbVr1xqcrrN06VK8/PLLmD17tkkWR6Y1tNVAnL59DjFXtuG1LtMhEomMGu9kJ4WDrTWP2CQiIiJqBIxO9GNjY/HFF1+gc+fOeOmll9CmTRsAwB9//IH169fjiy++QPPmzTFy5EiTL5Yej63EFsN9B+O7SzE4dfssQhTBRs+hkNvxiE0iIiKiRsDoGv1NmzYhKCgIGzZs0JXqtGjRAv3798e3336Ljh074rvvvquLtZIJhCiC0dKpBRJSdqFEXWr0eIXcHul3iqHlyTtEREREDZrRiX5KSgqGDBkCicTwjwESiQRDhgxBSkqKSRZHpicWiTHGbzgKlUXYfeOA0eO93OxRXKpG4T1VHayOiIiIiEzF6ETf2toa9+5VX6NdXFwMa2vrx1oU1S0fp+YIVXTFoZtHcbs4y6ixXrqTd1i+Q0RERNSQGZ3oBwYGYvPmzbhz547BtZycHGzZsgVBQUEmWRzVnWd8B0NmJUXsH9uNKsNRyCtO3uGGXCIiIqKGzejNuNOnT8ekSZMwZMgQjBo1SvdW3KtXryIuLg7FxcVYtmyZyRdKpuUodcDQVoMQ+8d2JN35HUHuHWo0romTDDJrK2Tc4RN9IiIioobM6ES/W7duWLVqFT788EN8/fXXete8vLywePFidO3a1WQLpLrzlHcofk4/ha1/7EDbJv6QWj265EokEsGTJ+8QERERNXi1Oke/X79+ePrpp3HhwgWkpaUBqHhhVvv27bFlyxYMGTIEu3btMulCyfSsxFYY7Tccn/76Pxz48zCGtBpYo3Fecjtc+jOvjldHRERERI+j1m/GFYvF6NixIzp27KjXfvfuXVy/fv2xF0b1w8/VF108OmJf6iGEeHaF3Nb1kWMUcnsc/z0TJWVq2Mpq/StERERERHXI6M24ZH5GPhEBEUSIu/pjjfpXbsi9ncsNuUREREQNFRN9gquNC8Ja9sO57N9wKfePR/b3cqs4YjOdG3KJiIiIGiwm+gQA6N/8KbjZNEHMlW0o15Q/tK+7iy2sxCJk8IhNIiIiogaLiT4BAKytrBHp9wxu38vC4bRjD+0rsRLDw9WWJ+8QERERNWA12kn54DGaD3P27Nka91Uqlfj000+xbds2FBQUICAgAHPmzEFoaOgjx2ZmZmLRokU4duwYNBoNevTogfnz56N58+Z6/dasWYOkpCQkJSXhzp07mDFjBmbOnGkw37x58xAfH2/QHhQUhC1bttT4nhqzDvK2aCf3x67r+9G1aWc4yxyr7eslt0caS3eIiIiIGqwaJfqLFy82alKRSFSjfvPmzcO+ffsQFRUFHx8fxMfHY8qUKdiwYQM6d+5c7bji4mJERUWhuLgY06ZNg0QiQXR0NKKiopCQkABnZ2dd3xUrVsDNzQ1t27bFTz/99ND12Nra4v3339dra9KkSY3uxRyIRCJEtnkGC09+gm0puxDVbmy1fRVudvj1jztQl2sgseIfhoiIiIgamhol+t9++63JAyclJWHnzp2YP38+Jk2aBAAYMWIEIiIisGzZMmzcuLHasZs2bUJqairi4uLQrl07AEDv3r0xbNgwREdHY/bs2bq+iYmJaNasGQoKCtCtW7eHrkkikWD48OGPf3ONWFM7d/Rv8RT2pR5CL+8eaO3sU2U/hdweGq0Wmbn34O3uUM+rJCIiIqJHqVGi3717d5MH3rNnD6ytrTF69Ghdm0wmQ2RkJJYvX46srCx4eHhUOXbv3r3o1KmTLskHAF9fX4SGhmL37t16iX6zZs2MWld5eTlKSkrg4GC5yWuYTz+cun0WMVcS8GbXmRCLDJ/Ye/11xGZGDhN9IiIiooZIE2DYHwAAIABJREFUsJqL5ORktGrVCvb29nrtHTt2hFarRXJycpXjNBoNLl++jA4dOhhcCwwMxI0bN1BSUlKrNRUXFyM4OBjBwcEICQnBRx99hLKyslrN1ZjZSGR41ncI/iy8hePpv1TZx7PJX0dsckMuERERUYMk2GtNs7Oz0bRpU4N2d3d3AEBWVlaV4/Ly8qBUKnX9Hhyr1WqRnZ2NFi1aGLUed3d3vPTSS2jbti00Gg0OHTqE6OhopKSkYN26dUbNZQ6Cm3bCT+knsP3aHnT2CISdtZ3edZnUCnInGx6xSURERNRACZbol5aWwtra2qBdJpMBQLVP0ivbpVJptWNLS0uNXs/rr7+u93NERASaNm2K9evX49ixY+jZs6fRc8rlwpS0uLtXf1qOMaZ2H4e5+xchMeMwXgg23Jjr4+WE7LxSvXimim0soeIKGZv3bP5xhYzNe7aM2JYWV8jYvGfLid2QCJbo29jYQKVSGbRXJvKVSfuDKtuVSmW1Y21sbEyyxhdeeAHr16/H8ePHa5Xo5+QUQaPRmmQtNeXu7ojs7EKTzGUPZ/T2CsXeq0fQpUlneDso9K67Ocpw4eodZGYVQCwSmTS2MYSKK2Rs3rP5xxUyNu/ZMmJbWlwhY/OeLSe2EMRiUbUPlwWr0Xd3d6+yPCc7OxsAqt2I6+LiAqlUquv34FiRSFRlWU9tuLm5wdraGvn5+SaZrzGKaD0Idta22HIlAVqt/pcWhdwOSrUGOfnG/wWFiIiIiOqWYIl+QEAArl+/juJi/c2c58+f112vilgshp+fHy5cuGBwLSkpCT4+PrC1tTXJGm/fvg2VSmVRZ+k/yN7aDs+0DsfVvOs4k3Ve75qXW8VG6nS+OIuIiIiowREs0Q8PD4dKpUJMTIyuTalUIi4uDl26dNFt1E1PT0dKSore2LCwMJw7dw4XL17UtV27dg0nTpxAeHi40WspKytDUVGRQfvq1asBAL169TJ6TnPypFd3NHf0RvzVnShV/713QnHfEZtERERE1LAIVqMfFBSE8PBwLFu2THdKTnx8PNLT0/HRRx/p+s2dOxenTp3C5cuXdW3jxo1DTEwMpk6dismTJ8PKygrR0dFwd3fXvXyrUkJCAtLT03X1+7/88osugZ8wYQIcHR2RnZ2NZ599FhEREWjdurXu1J3jx49jyJAhj3zRlrkTi8QY4zcCH5/5HHtTD2K472AAgIOtNZzsrHnEJhEREVEDJFiiDwBLlizBihUrsO3/2bvv+KjqfPH/r+kzyWQy6QkthEAIvUpRUNBFQYVlVUQBEetVd13dr+4Ff14fe6+713Xtu2tb26IIFjqKgFywRnogEUINnZBCept+fn9MMiZkAknIZELyfj4eeczMZ845789JJsn7c86nrF5NaWkpffv25Z133mHEiBEX3M9sNrNo0SKee+453nzzTTweD6NHj+bpp58mIiKi3rbLly9n+/btvtfbtm1j27ZtAEybNo2wsDAsFgsTJkwgLS2NlStX4vF46NmzJwsWLGDu3Lmtf+KXoV7hiYyOH8Hmk98zNuEKYkOiAe9V/bOS6AshhBBCtDtBTfQNBgPz589n/vz5jW6zaNEiv+Xx8fH84x//uGiMxvavy2Kx8OKLL150u87u18k3klGwl+WH1/DwkHsBSIgOZXtWXoOBukIIIYQQIriC1kdfXH7CDWFMSfoVewsPsPecd+XihKgQquwuyiobTncqhBBCCCGCRxJ90SwTul1FXEgsyw6vwelx0aVmQG6ODMgVQgghhGhXJNEXzaJVa5mRMo2C6kI2n/yehKgQAOmnL4QQQgjRzkiiL5qtX2QKQ2IGsv74JtBVY9RrOHtOrugLIYQQQrQnkuiLFrm1980oKKzK/oqEqBCZYlMIIYQQop2RRF+0SJQpkkk9JrArPwNzTJl03RFCCCGEaGck0RctNilxIpHGCPJMOyipsFFZ7Qx2lYQQQgghRA1J9EWL6TU6bu19M+VKEcbhm5m3+vf8bsOfWLLzm2BXTQghhBCi05NEX1ySfScKUBRQaV2oVKDoqvmxeIMk+0IIIYQQQSaJvrgkW4q+Q6WqX6bSePip8Nug1EcIIYQQQnhJoi8uiUdb3Wj5sdITKIrSxjUSQgghhBAA2mBXQFze1C4Tiq5hsq8CXtr1Bt3DunJ117GMjBuKXqNv+woKIYQQQnRSckVfXJIroyaguOt/jBS3mu6OK5mZ8htcHheLDyzj6bT/ZfnhL8ivOhekmgohhBBCdC5yRV9cklkjJ8JO+KnwWzzaatQuE2Elgzh0zMI13ZN4etQYjpQc4/szP/Ht6TQ2n/qBfpEpXNPtSgZEpaJWSVtTCCGEECIQJNEXl2zWyInMYiIxMWEUFJTjdLl59fMM3l+7H4New/CUXvSJ6EWpvYy0nG38eGYbb2cuJNIYwfguYxjb5QrC9OZgn4YQQgghRIciib5odTqthkdvHczLn+3h7dV7eey2IQxIiiTcYOHGpEnckHgtGef28cPpLaw+uo61x75mWOwQruk2lp6WHqjOn8anHdqem86a7PWU2EuwGqxMS57MqPjhwa6WEEIIIYSPJPoiIEwGLY/PGMILS9L554pMnpw5jN7dwgHQqDUMjx3M8NjBnK3M4/vTW9ieu4sdeemXxeDd7bnpLDmwHKfHuxJwsb2EJQeWA0iyL4QQQoh2QzpIi4Axm3Q8MXMoEWYDry7N4ERueYNtEkLjmNl3Ov971dPtfvCuR/FQYi9lxeEvfUl+LafHyZrs9UGqmRBCCCFEQ3JFXwRUuNnAk3cM46+Ld/HK53tYMHs4CVGhDbYzao1c3W0s47sGd/Cu0+2k0FbEueoiCqoLKax5PGcrorC6EKfH1ei+xfaSgNZNCCGEEKI5JNEXARcVbuTJO4bx/Me7eOnTPTw1ezjRVpPfbVUqFX0ivIN3S+ylpOVsJ62Rwbst6SevKAoVzkrOVRdxrrrwl0eb93mJvbTe9nqNnmhjJHGmaPpHphBjimLtsY1UOCsbHDvCYG35N0kIIYQQopVJoi/aRHxkCE/cMYy/LU7npU/3sGDOcKxmwwX3sRrCuSlpEpP9DN7tHtaNU+VncCneK+x1+8mPiB1Cka3El8DXvTJfWF2EzW2vFydcH0a0KYq+Eb2JNkUSbYqq+YokTGduMDjYqDXW66MPoFFpmJY8uTW+VUIIIYQQrUISfdFmusea+cPtQ3jp0z28/Oke5s8ejtmku+h+/gbvfn/mpwbbOT1OFmV9zqL9n+NRPL5yrVpLlDGSaFMkva29iDZFEmOK8pU1d9Bv7V2D2rsJGpUGBUi0dG/WcYQQQgghAkkSfdGmkruG8/tbB/Hq0kxe/XwPT94xDJOh6R/D2sG7/hJ9AA8eJve41ndFPtoURbjB0up9+0fFD2dU/HBiYsI4dOoUf93xGu/v/ZgnR/wOvebijRchhBBCiECTWXdEm+vXM5JHpg/kRG4Ff1+WicPpbvYxGusPH2GwMjV5MmO7XEGfiGQijNaAD+CNMFqZ228mZyrOsuzwmoDGEkIIIYRoKkn0RVAM7RPN/VP7cfhUCW+u2ovL7bn4TnVMS56MTl3/yrlOrQtaP/mB0f2Y1GMCaTnb2JG7Oyh1EEIIIYSoSxJ9ETRj+sdz1+S+ZGYX8s4XWXg8SpP3HRU/nFmptxJhsKLCeyV/VuqtQV2wamqvG+gV3pNPDi4nrzI/aPUQQgghhADpoy+CbMLQrtjsbj7/5ggL9RrmTUlFfd4sN42p20++oKDhYlxtTaPWcO+AWTy/4++8t/dj/jjyUemvL4QQQoigkSv6Iugmj+7B1Ct78mPmWT7bdARFafqV/fYmwmhlbv87yKnMZemh1cGujhBCCCE6MUn0RbswfXwSvxrRjY07T7H6x2PBrs4lGRDVl+sTJ/LT2e1sz00PdnWEEEII0UlJ1x3RLqhUKu74VR9sDjdr0o5jMmi5YVSPYFerxW5Oup7skuN8cnAFPcK6ER8aG+wqCSGEEKKTkSv6ot1Qq1TMm5LKyL4xfLb5CN9n5AS7Si2mUWu4d+As9God7+/9GIfbEewqCSGEEKKTkURftCtqtYoHpw1gYK9IPlx3gO3784JdpRazGsKZ1/9Ozlbm8bn01xdCCCFEG5OuO6Ld0WrU/PY3g3j1sz28+0UWBp2GIb2jg12tFukXlcINiRNZf2Izfay9GJ0wIthVarHtuemsyV5Pib0Eq8HKtOTJQZ3OVAghhBAXJlf0Rbtk0Gl4bMYQuseaeWPlXvafKA52lVrsxqRJ9LH24tODKzhbeXneodiem86SA8sptpegAMX2EpYcWC6DjYUQQoh2TBJ90W6ZDFr+38yhxEaY+MfyTLJzSoNdpRbRqDXcM2AWBo2B9/Z+jP0y7K+/Jns9To+zXpnT42RN9vog1UgIIYQQFyOJvmjXzCYdT8wciiVEx2ufZ3A6vyLYVWqRcIOFeQPuJK8yn88Prgp2dZqt2F7SaHmpPfiLlQkhhBCioaAm+g6HgxdffJFx48YxePBgbr/9drZs2dKkffPy8njssccYOXIkw4cP55FHHuHUqVMNtnvrrbd4+OGHueqqq+jbty///Oc/Gz1mdnY29913H8OGDWPUqFHMnz+foqKiFp+faB0RYQaevGMYep2Glz7bQ15RVbCr1CKpkX2Y3PM6tubuZMvZncGuTpN4FA9rj2284DZPp/2F19Lf5rvTP0nSL4QQQrQjQU30FyxYwIcffsi0adN4+umnUavVPPDAA+zevfuC+1VWVjJ37lx27drFQw89xO9//3uysrKYO3cupaX1u3e89tprZGZm0q9fvwseMzc3l9mzZ3Pq1Cn+8Ic/cO+99/LNN99w33334XQ6L7ivCLwYq4knZg7F41F46dPdFJXZgl2lFrkx6VekWJP57OBKcipyg12dC6p0VvFW5r/56thGki090al19d7XqXX8utcUJve8jnJHBZ8fWsXTaX/h1fS3+PZ0GqX2siDVXAghhBAQxFl3MjMzWbt2LU899RTz5s0DYPr06dx888289NJLLF68uNF9lyxZwokTJ1ixYgX9+/cHYPz48UydOpWFCxfy2GOP+bbdtGkT3bp1o6ysjCuuuKLRY7799tvY7XYWLVpEXFwcAIMHD+aee+5h9erV3Hbbba1w1uJSdIkO5YmZQ3nhk3Re/HQPC2YPJzxUH+xqNYtapWbegFn8dcervL/3Y/448lGMWkOwq9XAybLTvLt3EWX2Mu7oewvjuoxmR97uRmfdubnX9eRU5LI7P5P0gp9Zemg1yw6toVd4T4bHDmZo7ECshvAgn5UQQgjRuQQt0V+/fj06nY4ZM2b4ygwGA7fddhuvvvoq+fn5xMb6X010w4YNDB061JfkAyQnJzN27FjWrVtXL9Hv1q1bk+rz9ddfc+211/qSfIArr7ySnj17sm7dOkn024nE+DAenzGElz/bw58X7kABSsrtRFoM3HJNMmMHxAe7ihcVbghjXv87eX3Pe3x2aCVz+81EpVIFu1o+aTnb+PzQasJ0Zv4w4mF6WrwrFI+KH86o+OHExIRRUNCwi04XczxdzPHc1Ot6zlbmkZ6fye78TJYeXs2yw2tICk9keOxghsUOkqRfCCGEaANBS/T3799PUlISoaGh9coHDx6Moijs37/fb6Lv8Xg4ePAgM2fObPDeoEGDSEtLo7q6GpPJ1OS65OXlUVhYyMCBAxu8N3jwYNLS0pp8LBF4fbpZuW5EV9Zt/WVMRmGZnQ/XHQC4LJL91Mg+TEn6FV8d20gfazJXdmn8blNbcbidfH5oFVvO7iA1og/3DJiFWR968R39SAiN46akSdyUNIncyjx25/9Men4myw6vYdnhNfQKT2RY7GCGxQwiwmht5TNpHbJugBBCiMtd0BL9goKCelfPa8XExACQn5/vd7+SkhIcDodvu/P3VRSFgoICevTo0eS61MZq7JiFhYW43W40Gk2TjykCa3tWw8+Hw+VhxXfZl0WiDzCl53Vklxzj80MrSbR0o6s5IWh1OVddyHs/L+JURQ5Tel7HjUmTUKtaZwhPfGgcU5LimJL0K/Iq80nP/5ndBZksP/wFyw9/QZIlkeGxgxgWO7jdJP3bz+5iycEVvilFa9cNACTZF0IIcdkIWqJvs9nQ6XQNyg0Gb39lu93ud7/acr2+Yd/s2n1ttuYN1GzqMc+/+3AxUVHmZm3fWmJiwoISty1jF5X5/3wUltn57udcRvWPo3tcWJt0ibmUc37i6gf4zw3/y8L9S3h+0gKMOmObxK1rV87PvL7z3wAsGP8Iw7sMCljsmJgwBvZMZi7TySnPY+updLacSmf5kS9ZfuRL+kQlMbb7cMZ0G050aCQAP5zYzieZqymsKiIqJJI7B/+a8Ymjmh3b5XFTbq+g1FZOmb2cElsZZfZySm3llNrLKat5LLWVc66q4WxbTo+Ttce/5qZB17To3FsqWL/PneHvSHuJG8zYnS1uMGPLOXee2O1J0BJ9o9Hodzab2qS7NsE+X225w9Fw0aHafY3GpidLgTomQGFhBR6P0uz9LkVj/ac7WuxIi4FCP8m+Rq3iw7VZfLg2i+hwI0N6RzOkdxR9u0eg07b+JFOXfs4q7u53B//Y/S7/TPuIu/vf0aTGSWt8r2unzlx/fBPdzV24f9BconWRFz1ua/2cdYQwPmYc42PGkV9V4L3Sn5/JR3uW89Ge5fS09CDaGEnGub04PS4AzlUV8fb2jykrq+aKuGHY3DbKHRWUOyopd1bUPC/3va5w1JZVUOnyPy2rRqUhTG8mTBeKWW8m2RLtN9Gvjd+Wv1/B+n3uLH9H2kPcYMbubHGDGVvOufPEDga1WtXoxeWgJfoxMTF+u+cUFBQANDoQ12q1otfrfdudv69KpfLbBedCamM1dsyoqCjpttPO3HJNMh+uO4DD5fGV6bVq7p6SSmqPCDKyz5F5pJAfMnLYtOs0Bp2GAUmRDOkdxeDk6HY1W09KRG9uSprEl8e+pk9EL67qMjrgMSsclfx73xIOFB9mbMIV3J4yHb2m4R22thIbEsPkntcyuee15FedY3fNQN6d+XsabOv0OFmU9RmLDyzDVdMAOF+I1kSY3oxZZyY+NI6UiGTMejNhOrM3qa/90pkxaY0NGleHirP9LhKmUWkod1QQpg/O3TohhBCiOYKW6KemprJo0SIqKyvrdYnJyMjwve+PWq0mJSWFvXv3NngvMzOTxMTEZg3EBYiLiyMyMrLRY15sDn7R9mr74a/4Lpuisoaz7kwY2pUJQ7vicLo5cLKYPUcKyThyjvRD3sZcUoKFIb2jGJIcTY84c9Bnvbmh57UcKTnG0kOr6WnpEdD++sfLTvLezx9T7qxgduptXNml+d1gAik2JJobel7LDT2v5beb/9PvNh4Urus2DrM+tEHybtaFolVf2p+2acmTWXJgua+PPoBWpcGjeHhp5+s8MuRe4kL9X4wQQggh2ougJfqTJ0/mgw8+YOnSpb559B0OBytWrGD48OG+gbo5OTlUV1eTnJzs2/eGG27glVdeISsryzfF5tGjR9m6dSsPPPBAi+pz/fXXs2bNGvLy8nyxt2zZwvHjx7n//vsv4UxFoIwdEM/YAfEXvEWn12kYnBzN4ORolOtTOJVfQUZ2IZlHzrH6h2Os+uEYEWEGBid7k/5+PSMw6Nr+7o13fv07+ev2V3lv7yLmj/w9Rm3zu4tdiKIo/JizlaWH1mA1WHhi+CP0sDRt+tlgiTBY/V5ZjzBYmd77xoDFrR1we/6sOzGmaN7O/Dcv7XqDBwfdTZ+IXgGrgxBCCHGpgpboDxkyhMmTJ/PSSy/5ZslZuXIlOTk5/PWvf/VtN3/+fLZv387Bgwd9ZbNmzWLp0qU8+OCD3HPPPWg0GhYuXEhMTIyv0VBr1apV5OTk+Pra79ixgzfffBOAu+66i7Aw72CNhx56iPXr1zN37lzmzJlDVVUV77//Pqmpqfz6178O8HdDtAWVSkWPuDB6xIUx9cqelFU6yMwuJCP7HFuz8vhuTw46rZp+iRHevv3JUURaWjfZvpAwvZl7Bszi77vf4ZODK5jX/85Wu9PgcDv49OBKtuXuon9UX+b1v5NQXUirHDuQ/F1Z16l1TEueHPDYja0b8MeRv+PNjH/z+p53md1vhszCI4QQot0KWqIP8MILL/Daa6+xevVqSktL6du3L++88w4jRoy44H5ms5lFixbx3HPP8eabb+LxeBg9ejRPP/00ERER9bZdvnw527dv973etm0b27ZtA2DatGm+RD8hIYGPP/6Y559/npdffhmdTseECRN46qmn/M7GIy5/llA94wYnMG5wAi63h4OnSsg4co6MI+fIzC5kEdA91uzt4tM7mqQEC+qaxHvLvtxGuw1dij4Rydzc63q+OLqBPtZejOs65pKPmV91jvf2LiKnIpebkiYxued1rTZ1ZqA1dmU9mMl1tCmKJ0c8wjs/f8SHWZ9SWF3M5J7XBr37lxBCCHE+laIobTstTCcis+5cnnEVRSG3qIo9R86RcaSQI6dL8SgKlhAdg5KjMOm1fJeRg9PPQODWSPY9ioc3Mz7gcMlRnhzxO7qHdWmwTVPPOaNgHx9lfYZGpebuAXcyIKrvJdevo/ycLzWuy+Ni8YFlbM9NZ0z8SO5MveWSxwY0NXagyd+RzhG7s8UNZmw5584TOxja5aw7QrRXKpWKhKhQEqJCmTI6kYpqJ3uPFZJ5pJA9h89RaWs400trLtalVqm5u/8d/HX7a7y/dxHzr3gMUzP767s9br489jVfn/iGHmHduH/gXUSZIi6+o2gyrVrL3H4ziTZF8dWxjRTZS3hg4F2E6Jo3GYAQQggRKJfH/Xshgshs0jGmfzwPThvAa78f1+h2/ub1b6kwvZl7B87mXHURnxxYTnNuvJU7Kng9432+PvENV3UZzf8b/rAk+QGiUqm4KWkSc/vNJLvkGC+nv0lhtf85+IUQQoi2Jlf0hWgGjVpNVCOLdbX23Py9rUlM7XUDa46up7e1F1d3G3vRfY6WnuD9vR9T6axkTr/bGZswslXrJPwbnTCCCKOVd37+iBd3vc7Dg+8h0dI92NUSol3ZnpversbbiI5HPmMNyRV9IZrplmuS0ftZZbes0sGatGO4PR4/e7XMpMQJ9I/sy/LDazhVfqbR7RRF4dtTabya/hZalYYnRvxOkvw2lhKRzJMjHkGv1vFq+ttkFOwLdpWE8Gt7bjr/lfYcMz97mP9Ke47tuekBieP2uLG5vKtXbz75I0sOLKPYXoICFNtLWHJgecBii85ne246Sw4sl8/YeeSKvhDN5G+xrpuu7MmhkyWs+uEYGUcKuf/mfiREhV7kSBfn66+/4zXe2/sxC674PSZt/T7gdreDJQeWsTNvD4Oi+zG33x3STzxI4kPjeHLk73g7cyHv/vwRt/aZysTujXf3EqKt1SZDtVPWFttLWLx/GWfKz5JkTcTpduL0OHF4nL7nTo8Lp7umrE65o6bcWVPuqPPc6XHhUS580cPpcbIme32nv+IqWsfq7HX1pmIG+YyBJPpCtIi/xbomDO3K0D7RLNpwkP/59w5mTOzNxOFdfVNytpRZH1ozv/6/WHxgOfcNmO17L68yn3f3LiK3Mp+pvSZzfeKEy2bqzI7Kog/j8WH/wcKsT1l2eA0F1YXc1meq/FxEA63ZzcCjeCh3VFLmKKfMUU55zaP3eQVldu/zvKoCFOqP+XEpLv7v1Hdwyv+xNSoNOrUOnUaLXq2ree59NGj0hOpC0Gl0Ne9pfe/p62z3+aFVfo9dbC+h2FZChNHaovMWnZOiKORXnyO75DjZJcfILj1Gib3U77b+Fl3sTCTRF6IVjeoXR59uVhauO8DijYfYfbiAe2/sd8kLb9X211+dvY7/LDpMtauaUF0oNpcNo9bI74beT2pkn1Y6C3Gp9Bo99w+cw8oja9l86geKbEXcM2A2Bo2sySG8/F1ZX3JgOfDL+hEexUOVs9qXsPuS9vOf28upcFY2SOABDBo9Fn0YYfow4kPjyK3Kb7ROC654HH2dRF2n1qHX6FqlkbrxxLeNJlz/s/UFrutxDZN6TMCoNVxyLNHxuD1uTlfkkF1ak9iXHKfcWQFAqC6E5PAkKpxVVLuqG+yrQsWqI19xTbcrO2WDUhJ9IVpZRJiBx2cM5rs9OXy6+TDPvL+dOdenMKZ/3CUtqhSut6BCRVXNH7IKZyUqVEzueZ0k+e2QWqXm1j5TiTFF8fmh1byW/hYPDb6HcIMl2FUT7cCa7PV+uxksObCMzad+8CXx/rq/aNVaLPowLPowooyR9LT08L226M1YDGG+5P78xuV/pT3nN+GOMFj9rtnRWhpd5brXFE6Un2T98U38lLOdqb0mMyZhhNwB6+TsbgfHS09ypPQYR0uOc7TsBA63A4AoYwT9olJIDu9Jb2sSsSExqFXqBo1nAK1KS1dzPP938js2nfqeYTGDmNh9HEnhicE6tTYnib4QAaBSqZgwrCv9ekbw/pf7efeLLHYfKmDu5FTMJl2LjvnF0Q0NrtgpKGw6+b30A2/Hru52JZHGCN7ft5gXd77OI0PupYv50tdbEJcnh9tJVuGBRq9uOz0uLPowupm7EKY310ngf0nijRpjiy8aNJpwJ09u0fGa6mKrXE/odhXLD3/J4gNL+fb0j9zaeyp9I3sHtE6i/Sh3VHC09DhHSo6RXXqcU+Vn8CgeVKjoYo5nTPxIkq09SQ7v2ehV+Qt9xgqri/nuTBo/5WxnV34GPS09uLb7OIbGDEKj1rTlqbY5WRk3gGRl3I4dt6mxPR6FddtOsOqHY5hNOuZNSWVI7+hmx/rt5v9s9L03rn2h2cdrqc72c26tuKfKz/BWxgfY3U7uHzSHfpEpbRa7udr779TlFtflcXGg6DC78jPILNiHzW1HhcpvV5sIg5W/XPX/BaQetYI9BWFj32tFUUjPz2BV9jovV8tCAAAgAElEQVSKbMUMiu7Pb3rfRFxITMBjB1pn+p262OdLURQKbUVkl/yS2OfVdCnTqrUkhnUn2eq9Wp9kSWzR5BKNnbPNZWNr7i6+PfUjBdWFWA3hTOh2FVd1GUWILqTlJx1ksjKuEEGkVqu4aWxPBvWK4r0vs/j7skyuGdqFmdf2xqhv+q9ghMHa6C130f51D+vKH0c+ypsZH/Bmxgfc2fcWruwyKtjVEgHiUTwcLj7Krvw97MnfS6WrCpPWxLDYwYyIG0KpvYxPD65s8yvr4L3yOSp+eFCTT39UKhUj4oYyOHoA35z6kQ0nNvOXbS9zTdcrmZL0K0Iv40Sss2hs7ElRdQlGncHXv77UUQaASWsiOTyRMfEjSLYm0SOsKzpNy+56N4VRa2RCt6u4uutY9hUeYPOpH1mV/RVfHdvI6ISRTOx2FXGhsQGLHwyS6AvRRnrEhfHM3Vew6oejrN92kqzjRdx3U39SujctUQ/WLXfReiKMVv7fiEd4f+/HLD6wjHPVRdzc63rpj9xBeBQPx0pPsis/g/T8DModFeg1eoZED2BE3BD6RaagVf/yb1etUsviPn7oNDqu7zmRMV1Gsvbo13x7Oo1tubu4MWkS47uOqfc9FO1LY2NPvji2HgCrIZw+Eb1IDu9JsjWJhNC4oPz9U6vUDIruz6Do/pwuz+Gb0z+yJWc7P5zZwoCoVCZ2H0dqRJ9LGlfXXkjXnQCSrjsdO+6lxD50qoT3vsyisNTG5DE9mD6uFzo/i3CdL9i33KHz/ZwDEdftcfPZoZWk5WxnROwQ7up3u9+rWB3pnNt77JbGVRSFU+Vn2Jm/h/S8TIrtJWjVWgZG9WNE3BAGRqWiv8hsS5fbObd13DMVZ1lx+EsOFB8m1hTNb3rfxKDo/i1Kwi6Xc27vsZ0eFwVV58ityievMp/cqnxyK/M5XZHT6D7Pjn2KKFNEq9ajMS055zJHOT+c2coPp7dQ7qwgITSOid3HcUXccPQBvMvQGqTrjhDtTEp3K/9z7yg+23yEdVtP8nN2EQ9M7U/3WP+/qLXa6y130TwatYY7+95KtCmK1dnrKLGX8uDguzHrLn2RtUvRHhqSl4ucilx25WewK28PBdWFqFVq+kemMC15MoOi+2PSXtqUuuIXXc0J/G7o/ewrPMCKI2v5188fkmJN5pY+UwM6U5CAapeNvJokPrcmoc+rzOecrajejFCRxgjiQ2IxaAzY3fYGx4kwWNssyW8piz6Mm5ImcX3iRHbm7eGbUz+w5MBy1mSvZ1zXMVzddexlOWuaJPpCBInJoGXelFSG9olm4boDPLtwB7+5uheTR/VArb78bxeKC1OpVFyfOJEoYwQf7f+cl3e+wcND7iU2pPkDtVtDU+Z1D3T89t7IyK86R3p+BrvyMsipzEWFipSIZCYlTmBozCDpQx5AKpWKgdH96BeZQlrONr489jV/2/F3xiSMZGqvGy7LBCzQmvo7pSgKZY4K8qryfMl8bWJf25cevAunxYRE08WcwPC4IcSHxBIfGktcSIzvrpW/KS4vty6mOrWWsQkjGRM/gsMlR9l86gc2HN/MxhPfMiJuCBO7j6NHWLdgV7PJpOtOAEnXnY4dtzVjl1c5+GjDQXYdLKB3t3Duv6kfsRGNJw0d4Zwl7i+yS47zr58XAvDQ4Hn0Cu8ZkNgexUOls4oSexml9lLKHOWU2ssocZSx7exOnB5Xg33UKjXdzV0xaPToNXrfY+1zX7m6kfLa7dX6Rhdfaiw5mJV6a5sl+419r4tsxaTnZ7Irbw8ny88AkBzekxFxQxkWOwiLPixgsQPtco5b5axm/YlNfHsqDY1aw/U9JnBdj6ulm1SNxn6npibdQGxodL2r87lVBfUWmjJo9MSHxBEfGkt8SCxxobHEh8QQbYpq0lSU7aHR3trf7/yqc3x3Oo0tZ3dgdzvobU1iYvfxDI7u3y7GWF2o644k+gEkiX7HjtvasRVFYeu+PD7eeAiPR2Hmdb25ZkgXv/1QO8o5S9xf5Fed462MDyiyl3BVwigyz2U1+R+loihUOqsodZRRaq/5qve83FfmbwGmUF0Ilc6qRo/fLzIFh9uBw+3A7nHgcDuxu+3Y3Q6/x7sQvVrXoMFwpuIsLj+NjBCtiVv7TMWkNWLSGjFqjBhrn2uN6FphUKa/pKRvRB92F2SyKy+Do6XHAegR1o0RcUMYETuk1VfX7Oif7UDGLagqZFX2V+wp+BmrIZxfJ09hZNzQRpOv9jbVZEsoioLT48TudmBz2bG77dhqfh/tbjt2l50VR770La7YmDC9ueaqfFxNQh9DfEgsVkN4qwxC7Yj/p6qc1Ww5u4NvT6dRZCsmyhjJhO5XoVfrWX98E8X2EiKC0LiRRD9IJNHv2HEDFbuozMb7a/ez/0Qxg5OjmDclFau5/rLwHe2cJa5XhbOSl3e+QX71uXrlWrWWid3GER8aWz9xr0ney+xluBR3g+OFaE2EGyyE6y3eRz/PLYYwdGrtBVdMvdC87i6Py9sAqG0I1D731C+r9+hxYHd5t3G4HewvOtSi75dWrcWkMWLUGnwNgdpGQG2DwFte877WhElr8G23r/Agyw6vqXfVs+789l1C4xkRN4ThsUMC2qWqM3y2Ax33cPFRVhz5gpPlZ0gM684tfW6mtzWpTWI3xu9KrWot1/eYSLK1p6+xXJus2+sk694yR73yumX+1mBoqidGPEJ8SGzA543vyP+n3B43meey2HzqB9/FgLra+o6kDMYV4jISaTHyxB1D2bzrNEu/zeaZ97Yxd3IqV6R2rLl9RUNmXajf7jMuj4uNJ7/1vTZpjb6Evbc1yW8ib9GHNWumiJZO36pVa9GqtZeUNDTWyLAawnl82ENUu6uxuexUu2zYXDaq3TWPrl8evWV2CqoLveU1r5ubECkoGDVGnhjxiKxgfBnpE9GLP458lB25u1lzdD2vpr/FsJhBTO99I9GmqIDE9Cgeyh2VlDnqNrrLfQ3xfYUHcJ/XAHd5XHx1fGOjxzRo9Bg1Bgwag7cLnNZAmD6MmNrXGgMGrcHvdgaNoaZMzyvpb1FiL21w/AiD1dc1ULScRq1hWOwghsUO4qkf/0yZo36jwulxsiZ7fbsYZySJvhDtkFql4lcjuzMgKZL3vszirVV72T0gjpTuVtb+dJyiMjuRFgO3XJPM2AGSjHQk/hLeWv89Zj7hhrCL9kNuiQstHx9ojTUyfp08hZiQlidpHsVTcyW0plHgrm0seBsOSw4u97ufzW2TJP8ypFapGZ0wgqGxg9h08js2nviWn89lMaH7OGJMUaw/vrlJn223x025s6Je17eyOnfSvIl9OeXOika7woXrLQ2S/Lr+MPzhXxL3muS8sTEsLfHr5CmX/aDYy8X5SX6tC/0tb0uS6AvRjiVEhfLUnBF8teUEq348xtZ9eb73CsvsfLjuAIAk+x3IhVZAvpSktymCNX1roBoZapXa133H38R+62r61J5PVpu+vBk0em5MmsSVXUbxRfYG/u/kd/XeL7aXsHj/Ug4WHcZqtJ53Nb6MCkdlgztBKlSYdaFYDGGEGyx0NXchXB+GxXcnLQxLna5w0PidqgiD1W+3otYUzIZ7Z9PeV62XRF+Idk6rUTNtXBLf7D5DaaWj3nsOl4fPNh0mtUcE4WY96g6wil9n11lXQA5GI6Ozfq87C6shnLv6305W0cEGV11diputubtQocKiN2MxWLAawkm0dMOi/yV5r+0GZ9GHNWnGmbqC/fmSdVfaRrB/zhcjib4Ql4nzk/xaZVVOnngjDa1GTVS4kRirkZhwE9E1jzFW7/NQY/te2U94yZW4tiPf686hsa4VAP+Y+NeATY8on6/Ooe7POViz7lyIJPpCXCaiLAYKyxquOBgWomP6uCQKSm2cK6mmoMTGsZwyKm31B3WGGLQNkv8Yq4nocCPR4UZ02gtfrdqyL5cV32XL+IA2IFfi2o58rzu+C3WtCPQc6PL56hxqf87tkST6QlwmbrkmmQ/XHcDh+mXwl16r5o7r+vhNuKtsTgpKbJwr9Sb/BaXVnCuxceZcJRnZhbjc9QeRRYQZapJ+k/euQE0jIMZq4sDJYj5af9AXW8YHCCEuF+29a4UQgSSJvhCXidqEuqlX1UOMOhLjdSTGN1y506MolFY4KCip5lxNA6CgpJqCUhsHTxWzdZ/9opMSOlweVnyXLYm+EKJdky40ojOTRF+Iy8jYAfGMHRB/ybeB1SoVEWEGIsIMpHRvODOA0+WhqMx7F6CgxMaiDQf9HqewzI6iKK2yiqIQQgSKdKERnVVgO6cJIS5LOq2auMgQBiZFMXFYV6Ishka3/a/3trF+20nKqvwPFhZCCCFEcEiiL4S4qFuuSUavrf/nQqdVM35wAiFGLZ9/c4QnXk/jzZU/8/PRQjyeli/PLoQQQojWIV13hBAXdbHxAWcKKvgh8yw/7c1l58ECIi0Gxg1KYNzgBKLDTcGsuhBCCNFpSaIvhGiSC40P6Bpj5o7r+nDrNcnsOXKO7zNy+CLtOF+kHad/UiRXD+nC0N7R6LRyE1EIIYRoK5LoCyFajU6r5orUWK5IjeVcSTU//nyWH38+y1ur9mI26bhyYDzjByfQNcYc7KoKIYQQHZ4k+kKIgIi2mpg+vhfTrkpi3/Eivs/IYdOu03y94xTJXS1cPbgLV/SLxaiXP0NCCCFEIMh/WCFEQKnVKgb1imJQryjKKh38tDeXHzJz+Pe6AyzZdJjR/WIZP6QLvRIsMk2nEEII0Yok0RdCtBlLqJ7Jo3tww6juZJ8p4/uMHLZm5fF9xlm6xoQyfnAXxg6IIyxEH+yqCiGEEJc9SfSFEG1OpVLRu1s4vbuFc+ev+rBtfx4/ZJzl002HWfbtEYanxDB+SBf6JUawLSuvyasBCyGEEOIXQU30HQ4Hf//731m9ejVlZWWkpqbyhz/8gbFjx15037y8PJ577jnS0tLweDyMGTOGp556iu7duzfYdunSpXzwwQecPn2aLl26MHfuXGbPnl1vm3/+85+8/vrrDfaNjo4mLS2t5ScphLggk0HLhKFdmTC0K6fyK/ghI4ct+3LZvj8fs0lLtd2Nu2Ze/sIyOx+uOwAgyb4QQghxEUFN9BcsWMDXX3/N3LlzSUxMZOXKlTzwwAMsWrSIYcOGNbpfZWUlc+fOpbKykoceegitVsvChQuZO3cuq1atIjw83Lftp59+yp/+9CcmT57MPffcw86dO3n22Wex2+3ce++9DY797LPPYjQafa/rPhdCBFb3WDOzJqUwY2Iy6YfO8f7a/b4kv5bD5eHjrw9SZXMRHqon3Kwn3GwgPFSPQacJUs1b15Z9uXIXQwghxCULWqKfmZnJ2rVreeqpp5g3bx4A06dP5+abb+all15i8eLFje67ZMkSTpw4wYoVK+jfvz8A48ePZ+rUqSxcuJDHHnsMAJvNxquvvsp1113H3//+dwBuv/12PB4Pr7/+OjNmzCAsLKzesadMmYLFYgnAGQshmkqn1TC6fxz/WrPP7/vVdjeLNx5qUG4yaLCEGrDWNgBCDVjNeiyheqw1jYFws55Qkw71RQb+BivZ3rIvlw/XHcDh8gByF0MIIUTLBS3RX79+PTqdjhkzZvjKDAYDt912G6+++ir5+fnExsb63XfDhg0MHTrUl+QDJCcnM3bsWNatW+dL9Ldt20ZJSQmzZs2qt//s2bP54osv+P7777npppvqvacoChUVFYSGhsoMIEIEWZTFQGGZvUF5pMXAM3dfQWmFndJKByUVdsoqHZRUOCitdFBaYed4bjmlFYXYne4G+2vUKiyhesJrGgDehoC+piFg4FReOV9tO4nzvGRbURRG9YvD5fbgciu4PQput8f32uX21JR5n7s8nprnCm6Pp95+Lrfnl+3q7PftnjO+JL+Ww+VhycZDGHUaQoxaQk06Qo06Qoxa9Fp1q/2tkjsJQgjRsQQt0d+/fz9JSUmEhobWKx88eDCKorB//36/ib7H4+HgwYPMnDmzwXuDBg0iLS2N6upqTCYTWVlZAAwcOLDedgMGDECtVpOVldUg0Z8wYQJVVVWEhoZyww03MH/+fKxW66WerhCiBW65Jrne1W0AvVbNrdcke5Py0IvPzmNzuCitaQCU1DQMSiscvkbCuVIbR3NKKa9yolzgOA6Xh/e+3M97X+5vhTNrSKNWodGocDg9ft+vtLn454qfG5RrNSpf0h9q1BFq1BLie6xtFPxSVnebuisVy50EIYToeIKW6BcUFBAXF9egPCYmBoD8/Hy/+5WUlOBwOHzbnb+voigUFBTQo0cPCgoK0Ov1DRL12rK6MSwWC3fddRdDhgxBp9OxdetWPvvsM7Kysli6dCl6vUz3J0Rbq00wL+Uqs1GvxRipJS4y5ILbudweyquclFbaeXbhzka3u+XqXmg1ajRqFVqNCo1GjVaj8pX5XqvVaOqUa2vKNWp1g/3UapWvK9Ef30zzexfDatbz2G1DqLQ5qbS5qLQ5qbK5qKz2vq6qKS+usHPmXCWVNhfVdtcFz1mvVRNq8jYS8oqqcLkbjodY+k02o/vHXbSrkxBCiPYnaIm+zWZDp9M1KDcYDADY7Q3/0dUt95d41+5rs9kuGKN227ox7r777nrvT548mT59+vDss8+yatUqbr/99oudUgNRUeZm79MaYmLCLr5RB4st59xxY0+bEMa0CX3aJFZCzWNMxD4KiqsbvB8TYeKeXw8KaB3m3TyA15dm1OtyZNBpuG/aQEYO6tKsY7k9CpXVTiqqHVRUOb1f1Q4qqr3Py6scNe87OVNQ6fcYJRV2Hn75O+KjQukSHUpCdN1HM1FWExp16zYC5HdK4nbE2HLOnSd2exK0RN9oNOJ0OhuU1ybftUn7+WrLHQ5Ho/vWzpRjNBr9ble7bWMxat155528+OKLbNmypUWJfmFhBR7PhToDtL6YmDAKCsrbNGawY8s5d47YbRl3+rgkv12Gpo9LCngdBvSwMndy3wZ3MQb0sLY4tg6IMGmJMGkBk99tDp0o8nsnIdSoZfyQLuQVVXE6v5z0g/m+sQvg7ToUYzURFxFCbISJuAgTsZEhxFlNRFqMqJvZCOgMn6/2EruzxQ1mbDnnzhM7GNRqVaMXl4OW6MfExPjtnlNQUADQ6EBcq9WKXq/3bXf+viqVytetJyYmBqfTSUlJSb3uOw6Hg5KSkkZj1FKr1cTFxVFaWtrk8xJCXP5ao8vQpcYfOyC+Tf9ZNTYeYtaklHrn7VEUSsrt5BVXk19cRX5xte951vGievvXNgJirSbiImsbAt7HqPMaATIQWAghWl/QEv3U1FQWLVpEZWVlvQG5GRkZvvf9UavVpKSksHfv3gbvZWZmkpiYiMnkvWLVr18/APbu3cu4ceN82+3duxePx+N7vzFOp5OzZ882GMwrhOj4gpFsB1NTGzdqlYpIi5FIi5F+iRH13vMoCqUVDvKKqsgvqfY+FleTV1zF/hPF9RoBGnVNIyDChMejsP9EsSyMJoQQrSxoif7kyZP54IMPWLp0qW8efYfDwYoVKxg+fLhvoG5OTg7V1dUkJyf79r3hhht45ZVXyMrK8k2xefToUbZu3coDDzzg227MmDFYrVaWLFlSL9H/5JNPCAkJ4eqrr/aVFRUVERkZWa+O77//Pna7nfHjx7f6+QshRHtzqY0btUpFRJiBiDADqec1AhRFoaTCQX5xFXk1yX9+cTV5RdWcLqhocCyHy8MHa/ezY38+ERYDkWEGbwMjzECUxYg1zIBWo26wnxBCiF8ELdEfMmQIkydP5qWXXvLNkrNy5UpycnL461//6ttu/vz5bN++nYMHD/rKZs2axdKlS3nwwQe555570Gg0LFy4kJiYGF+jAbx99H//+9/z7LPP8thjjzFu3Dh27tzJmjVrePLJJ+stjDVx4kRuvPFGUlJS0Ov1bNu2jQ0bNjBixAhuvvnmNvmeCCFER6Wq0wjo26N+I+De5zf73cftUThXauPw6RIqbfVnEFIBFrOeyDAjkRaD7zHKYqxpGBgJD9VfdIyAdBkSQnRkQUv0AV544QVee+01Vq9eTWlpKX379uWdd95hxIgRF9zPbDazaNEinnvuOd588008Hg+jR4/m6aefJiKi/j+Q2bNno9Pp+OCDD9i0aRMJCQk8/fTTzJ07t952U6dOJT09nfXr1+N0OunatSuPPPII//Ef/4FWG9RvkxBCdGiNLYwWZTHw7H2jAO96CEVldorKbd7HMhtF5XaKy2zknKvk56OFDdYg0KhVWM0Gb0Og5m6At9uRtyFw9Gwpn206ImsHCCE6LJWiKG07LUwnIrPudOy4wYwt59zx4wYzdlvHPX+xLvAOBL57SmqTE25FUai0ueo1AIrK7RSW/dIwKC63+8YBXIhRr2HSyO6YDN5Fx0IMWkw1jyEGra/8UrsOBftuQmf5fLWH2HLOnSd2MLTLWXeEEEIIaJ1ZjlQqFWaTDrNJR484//NnexSF8koHReXexP+NlQ0ndQCwOdx8+dPxC66UDKDTqusl/qa6DYFGGge17+09Vsgn/3dY7iYIIQJKEn0hhBBB1xazHKlVKsLNBsLNBpISLBfsMvS3h6/EZndTZXdSbXdTbXdRVbPacFXNV7WtzvOa9wtLbb7XddcbaAqHy8OSjYeIshiJiwzBEqJDJSsSCyEugST6QgghOqXG1g645Zpk1CqVt9uOseX/Jp0uj69hcH5DYWHN1fvzVdpcPL84HfB2IYqLCCEu0kRsRAhxtesQRJoIM0kjQAhxcZLoCyGE6JQCvTCaTqtGp9VjCdU3eO+LtGN+7yZYzXrmTUklr6jatwbBsbNl7DiQT90RdSaD1pv4R9ZvAMRFhGA26S5Yr2CNDQj2mAQhOiNJ9IUQQnRawVoYrbG7CTMm9mZwcjQk19/e5fZQUFKzCnFRFXkl3sfsM6Vsz8qrN54g1Kj13gGoSfzrNggysgvrxW2rsQHnD7iWMQlCtA1J9IUQQog21ty7CVqNmoSoUBKiQhu853TVNgKqvHcCalYlPnyqhG376jcCVCo4f649h8vDx18fJK+oCpVKhUrlHdysrnlUqUCFCrXa+1xdZxvfa+q8rred9726A4/rxl32bTaj+sWiUcviZ0IEgiT6QgghRBC01t0EnVZNl+hQukQ3bAQ4nG7fnYC84iqWfpPt9xjVdjdr0o63uA4tVVxu54EXvsVk0BBq1Hm/TNqaRx2hRq2vzFxTFlJTZjZp0Wk1TYoj3YZEZyWJvhBCCNFB6XUausaY6RrjnWN7867Tjc409OIjV6EoCorinYpUUbzrE9R/jm+b8197t2u4v6LAy5/tobTS0SBuiFHL9SO7U2FzUlntotLmpNLmpKjM7n1e7cJzgeV+9Fr1eQ2Cmud1HnMKKvl2zxlcbu9x2rLbUGdsYHTGc27PJNEXQgghOokLzTQEdbrf0Loz+tx+bW+/cWdPSrlgEqgoCjaH25f0exsCLiqrnb4ybyPBW55XXEVltZOKahcud+PTmzpcHt79IotP/u8wRr0Go16LyVD30fvc92jQYPK91mAy1Dw3aDHpNWg16gazIHXGcQmd8ZzbO0n0hRBCiE4i0DMNtXZclUqFqWbBsejw5sV0ON1U2lw88UZao9tckRqLzeHC5vCulVBe5aCgxE11TZnd4W5SLI1a1aBRcCK3HKe74biEzzYdpleCBWuYAYOuaV2P2iuny01xuZ2iMjvF5XY+3njI71iMT/7vMPGRIcRGmAg1XnhWKNG6JNEXQgghOpFgzTTU1nH1Og16neaCC6PddUPfCx7DoyjYHW5fQ8DmcGNzuKi2u30NhLoNBVudbc9P8muVVTl56p2tgHea1IgwA1azngizAWuYAau55ivMW2YJ1aPVNH+w8qV2obE73BRX2Ckus3lXky73JvPFZTZvcl9up6La2aRjVVQ7+fOHOwEIMWiJiTARYzURYzUSa619biLSYpCB2a1MEn0hhBBCdFgX6650Ieo6dxQiwgzNivvHN9P8NjAsITpmTOxNSYWdknIHJRV2iivs7D9ZTGmFA7en/pgEFWAJ1WM1G3yNgtoGQUSdx1Cj1td96GJdaKrtLm/SXm6nqNxW76p8cc3rSpurQd3NJh2RYd54vbqGExFm8L2OCDPw8qd7KCpveM7hoXrmXN+XgpJqCkqrKSiu5lReObsPFdQ7X41aRZTF2GhDwGS4cNoq4wMakkRfCCGEEB1WsLorNdbAmHldn0ZjexSF8ionJeXe5N/bGPA+Fpc7KCyzceRMqd8r6VqNquZOgIGTueV+u9B8sHY/H399kGp7wy5JlhAdEWFGosNNpHS31iTxRm8SbzEQYTagv0hXo1sn+D/n26/tzYi+MQ3P16NQVG6joMTmbQTUfOUXV3P8bFmDxobZpCPGaiI2wtsIiLGafA2BAyeL+Wj9QRkfcB5J9IUQQgjRoQWju1JLGhhqlYrwUD3hoXoSCWt0O6fLQ2mFnZIKh7dBUNsYqHl+fpJfy+1RuHJggvcqvOWXRN5qNqDTXnqXmeaes1qtIjrcRHS4iX6JEQ3er7I5fY2A/DoNgewzpezYn3/BGZnA27hZsvEQJr0Wc4iOsBAdYSYdJoO2weDpS1F7J6GwzE5UO7uTIIm+EEIIIUQABKqBodOqibaaiLaa/L7fWLehKIuB2ZNSWq0e/rTmOYcYdSTG60iMb9jocbk9FJXbKSj2Jv8fbTjo9xiVNhf/WJ5Zr0yjVhFq8ib9ZpO3AWAO0Xufm3R1GgXeMnOIrtGB0+19piFJ9IUQQgghOpBLGZdwudBq1MTWdN0BWLvluN/GjdWs59FbB1Ne5aSi2kFFlZPyamfNaycVVQ7OnKukorqEimpng5Wja+m1am+DwKT3NgRqGglpe8/67Sa14rtsSfSFEEIIIUTrCta4hGBqrHEzY2JvkhIsTTqGx6NQVTPNqrcRUNsoqP+6otpJfjh5m/gAABu2SURBVHEVFdVOv+MdAL+NjmCQRF8IIYQQooMJ1jSqwdIajRu1WuXtqmNq+lz/T76ZRlEj3aTaA0n0hRBCCCHEZS8YjZtb23k3KUn0hRBCCCGEaIG6dxJk1h0hhBBCCCE6kNo7Ce2RrDMshBBCCCFEBySJvhBCCCGEEB2QJPpCCCGEEEJ0QJLoCyGEEEII0QFJoi+EEEIIIUQHJIm+EEIIIYQQHZAk+kIIIYQQQnRAkugLIYQQQgjRAUmiL4QQQgghRAckK+MGkFqt6lRxgxlbzrlzxO5scYMZW865c8TubHGDGVvOufPEbmsXOleVoihKG9ZFCCGEEEII0Qak644QQgghhBAdkCT6QgghhBBCdECS6AshhBBCCNEBSaIvhBBCCCFEBySJvhBCCCGEEB2QJPpCCCGEEEJ0QJLoCyGEEEII0QFJoi+EEEIIIUQHJIm+EEIIIYQQHZAk+kIIIYQQQnRA2mBXQFy6/Px8PvroIzIyMti7dy9VVVV89NFHjB49OqBxMzMzWblyJdu2bSMnJwer1cqwYcN4/PHHSUxMDFjcn3/+mbfffpusrCwKCwsJCwsjNTWV3/72twwfPjxgcf159913eemll0hNTWX16tUBi7Nt2zbmzp3r972vvvqK5OTkgMWulZmZyeuvv87u3btxuVx0796defPmccsttwQk3oIFC1i5cmWj73///ffExcUFJPbx48d57bXXSE9Pp6ysjC5dujB9+nTmzZuHXq8PSMxae/bs4dVXXyUzMxO1Ws3o0aNZsGABPXr0aLUYzfmbsWnTJl5//XWOHDlCVFQUt912Gw899BBabfP/fTQ17ieffMLWrVvJzMwkJyeH3/zmNzz//PMBPd/i4mKWL1/O5s2bOXr0KC6Xi+TkZObNm8eUKVMCGltRFP70pz+xe/duzp49i9vtpnv37tx2223ceeed6HS6gMQ935kzZ7jxxhux2WysWrWKfv36BeycAa699lrOnDnTYP8HHniAJ598MmBxAcrLy3njjTfYsGEDBQUFREVFMWLECF555ZVmx21q7Av9HQd4/PHHefjhh1s9LoDdbuff//43q1ev9v2/HjlyJL/73e9ISkpq3sk2M3Z5eTmvvPIKGzdupLS0lKSkJB544AGmTp3a7JjNyTvS09N58cUXycrKwmw2M2XKFJ544glMJlOLzvdyJYl+B3Ds2DHeffddEhMT6du3L7t3726TuO+99x7p6elMnjyZvn37UlBQwOLFi5k+fTrLli0LWPJ56tQp3G43M2bMICYmhvLycr744gvmzJnDu+++y1VXXRWQuOcrKCjgrbfeIiQkpE3iAdx9990MGDCgXlmgkt26vvvuO377298yatQoHnvsMbRaLcePH+fs2bMBizlz5kzGjh1br0xRFP77v/+brl27Buy88/LymDFjBmFhYcyZM4fw8HB27tzJyy+/zOHDh3nxxRcDEhe8/8TmzJlD165defTRR/F4PCxZsoRZs2axatUqoqOjWyVOU/9m1P7cx4wZwzPPPMOhQ4d44403KC4u5plnnglY3HfffZeKigoGDRpEQUFBs+O0JO6ePXt47bXXuPrqq3n44YfRarVs2LCBxx9/nKNHj/Lb3/42YLE9Hg/79u1j3LhxdOvWDY1Gw549e3juuefYu3cvL7zwQkDinu9vf/sbavWl3+hvTuwBAwZw99131ytLSUkJaNyysjJmz55NWVkZM2bMID4+noKCAnbs2NGiuE2NnZyc7PdnuWbNGn788ccW/e9q6jn/8Y9/ZNOmTdx+++3079+f3NxcFi9ezI8//shXX31FVFRUQGK7XC7uueceDhw4wJw5c+jRowc//vgjTz75JG63m+nTpzcrZlPzjv379zNv3jx69+7NggULyM3N5YMPPuD06dO8/fbbzT7Xy5oiLnvl5eVKUVGRoiiKsnHjRiUlJUXZunVrwOPu2rVLsdvt9cqOHTumDBw4UJk/f37A49dVVVWlXHnllcqDDz7YZjHnz5+v3HXXXcqcOXOUadOmBTTW1q1blZSUFGXjxo0BjeNPWVmZMnbsWOXPf/5zm8c+344dO5SUlBTlrbfeCliMf/3rX0pKSopy6NCheuWPPvqo0r9/f8XhcAQs9n333aeMGjVKKSkp8ZXl5eUpQ4cOVf7yl7+0Wpym/s248cYbld/85jeKy+Xylb3yyitKamqqcuzYsYDFPX36tOLxeBRFUZQRI0Zc8t+TpsQ9efKkcvr06XplHo9HmTt3rjJ48GCluro6YLEb8+c//1np27evUlhYGPC4W7duVQYMGKC88sorSkpKipKVldXsmM2NPXHiROXhhx9ucZyWxn3mmWeUa6+91rdtW8b2Z9KkScr1118fsLgFBQVKSkqK8vzzz9cr37x5s5KSkqIsW7YsYLHXrl2rpKSkKCtXrqxX/uijjypjx45tkENcTFPzjvvvv18ZP368UlFR4Sv7/PPPlZSUFOWnn35qVszLnfTR7wDMZjMRERFtHnf48OENujH07NmTPn36kJ2d3aZ1MZlMREZGUlZW1ibxMjMzWbNmDU899VSbxKuroqICl8vVZvG++OILysrKeOyxx3zxFUVps/h1ffnll6hUKm6++eaAxaisrARocIUrOjoarVaLRqMJWOz09HTGjRtHeHi4ryw2NpZRo0axbt26VovTlL8ZR44c4ciRI8ycObPeOc+aNQuPx8PXX38dkLgAXbt2RaVSNfv4lxK3e/f/v707j4rqPOM4/kVEExVQIkkMYII2agAFDhGDGDdAKUqxMSmyVYXgQmJdojkkJjE1WntS9LSCUKrGaqtiXVg1NSoYBNQcmxA0bNVEgRNBFkdBkEGY/uFhjjhYR5w7RvJ8/pt3LvzuZXnvc++873vtsLGx6dBmYmKCt7c3N2/e7HSIiaGy7+W5555Do9FQX1+vaG5raytr164lNDTUIMMuH/SY1Wo1TU1NRsm9fv06ycnJREREMGDAAJqbm1Gr1UbJ7kxBQQGXLl3q0jAWfXMbGhoAdD4RbH/9xBNPKJb99ddfY2JiojP8zc/Pj9raWk6fPv1AmfrUHQ0NDeTl5TFjxgz69u2r3S4gIIA+ffoYtC99HEihLwxKo9FQU1NjlAuPhoYG6urq+P7779mwYQOlpaU6Qz2UoNFo+OSTT5gxY0aXx7B21YoVK3Bzc8PZ2Znw8HBKSkoUzzx58iRDhgzhyy+/ZMKECbi5ueHu7k5MTAytra2K57draWnh888/x9XVFVtbW8VyRo8eDcDKlSspLi7m8uXLpKWlkZycTGRkpEGGNtyLWq2md+/eOu1PPPEE1dXVXLlyRbHsuxUWFgLg5OTUof2ZZ57h2Wef1b7f3dXU1AAYpU9raWmhrq6Oy5cvc+TIET777DPs7OwU/XsHSEpKoqqqiqioKEVzOpObm4uLiwsuLi54e3uzZ88eRfPOnDmDWq1m4MCBzJkzB2dnZ1xcXAgPD6esrEzR7M6kpaUBdLnQ14etrS2DBg1i27ZtZGZmUllZSX5+PmvXrmXo0KF4eXkplq1Wq+nZs6fOPJP2cfKG6EfurjtKSkq4deuWTt/Vq1cvXnrpJYqKih4683EiY/SFQaWlpVFVVcXSpUsVz3r//fc5fPgwAGZmZsyaNYsFCxYonpuSksL58+fZtGmT4lntzMzMmDp1KuPHj2fAgAGUlJTw2WefERwczL59+7o8mUofly5dorKykujoaN58800cHBzIyspi8+bNNDc3s3LlSsWy75STk4NKpVL0hAgwbtw4Fi9eTGJiIpmZmdr23/3ud10ep60ve3t78vPzaWtr015QqNVqCgoKgNuT355++mlF96Fd+9h4a2trnfesra2NetHxqKhUKvbu3Yu7uztWVlaK5+Xk5HTow5ycnFi3bp2inyKpVCo2btzIokWLsLCwUCynM8OGDePll1/mhRde4OrVq/zrX//io48+4tq1a8ybN0+RzPZi/sMPP8TJyYkNGzZw5coV4uLimD17Nunp6fTr10+R7Lu1trby+eefM2rUKEUXsOjZsycbN27knXfe6TDZ18XFhX/+859dvqOvD3t7e1paWigoKMDFxUXbfubMGQCD9CN31x3367vy8/MfOvNxIoW+MJgLFy6wevVq3NzcCAgIUDzvrbfeIjAwkMrKSlJTU1Gr1bS0tCi6KkpDQwPr169n3rx5Riu44PbHlXeuKOTl5cXkyZOZOXMmcXFxrF+/XrHsxsZGrl27xjvvvKM9+U6ZMoXGxkZ2797NwoULjVIEZWRkYGZm9lAroOjL1tYWd3d3fHx86N+/P8ePHyc2NhYrKyuCgoIUyw0ODubjjz/mgw8+IDw8nLa2NhISErQnrps3byqWfbf2rM7+n3r37m2QoRY/ZW1tbSxfvpz6+no++OADo2Q6Ozuzbds26uvrOXXqFEVFRTQ2NiqauXHjRqysrJg1a5aiOZ25e1Lka6+9RnBwMPHx8QQFBWFubm7wzPahedbW1mzevFl7QW1vb8+8efPYv3+/zuRgpZw8eZKamhrmz5+veJaFhQUvvfQSv/zlLxk1ahRlZWUkJiayePFitm7dqth5c/r06WzatIno6Gg++ugjBg8eTG5uLrt27QIevk/rrO64X99lzH70p0CG7giDqK6uZv78+VhaWvKXv/xF0eEN7YYPH46npyczZ85k69atfPfdd4qPmU9ISMDMzIy5c+cqmqOPESNG4OHhwalTpxTNab/bc/e4eH9/f1paWjh79qyi+XD75Hzs2DHGjRun+BCKgwcPsmrVKtasWcNvfvMbpkyZwh/+8Ad+/etf8+mnn3Lt2jXFsoOCgliwYAFpaWlMmzYNf39/ysrKiIiIAOgw3lRp7b/3zsYvNzc3K3oX8Kfgk08+IScnh3Xr1jF8+HCjZFpZWTF27FimTp3KqlWr8PLyYu7cuQZZeagzpaWlJCUlER0d3aXlUg3N1NSU2bNn09TUpNjqce1/t76+vh3OUxMmTMDS0pKvv/5akdzOpKenY2pqip+fn6I59fX1hISE4ObmxrJly/D29iY8PJzY2Fi++uorUlJSFMu2trYmISGB5uZm5s6di5eXF59++ql21a6HWbXuXnXHz73vupsU+uKh1dfXExkZSX19PVu2bOn04zKlmZmZ4eXlxRdffKHY1fqVK1fYvn07wcHB1NTUUFFRQUVFBc3NzbS0tFBRUaFoEdiZQYMGKZ7Z/vu810QuYxzz0aNHaWpqUnzYDsCuXbtwdHTUWb5z8uTJNDY2UlxcrGj+0qVLyc3NZefOnaSlpbF//340Gg0mJibY2dkpmn2n9t97Z0VmdXW1UT/RMra4uDh27drFihUrFJ34fT++vr40NjZy7NgxRb7/hg0bcHBwYOjQodr+7OrVq8Dt/k7J5XPv5dlnnwWU61fu1Z8BRl3Q4ebNmxw5cgQPDw+DLZt7L4cPH6ampobJkyd3aHd3d6dfv36KX9yMHj2ao0ePkpKSwq5du8jOzsbZ2Rm4PZG2K/5f3fFz7rs68+gv4cVjrbm5mQULFnDx4kX+/ve/M2TIkEe2Lzdv3kSj0XDjxg1Frthra2tpaWkhJiaGmJgYnfe9vLy6/KCXriovL1f8DrejoyN5eXlUVVV1KDQrKysBjDJsJz09nT59+uicqJRQU1PT6TG1tLQAGGUCsqWlJS+//LL2dV5eHqNGjTLa2GFAO9H83LlzHZ7dUFVVRWVlpdEnohvLzp07iY2NZc6cOdpPUh6V9psWXVl1Rx+XL1+muLi408mY8+bNY+DAgeTm5iqSfS/l5eWAcv1K+99yVVVVh/a2tjaqq6t1nlOilMzMTG7cuGGUmxe1tbXA7WO8k0ajoa2tzSiruJmamnboM/Ly8gB45ZVXHvh73a/uGDZsGD179uTcuXNMmTJF265WqykqKjLKz/ynRAp90WWtra0sWbKE/Px84uPjO0y0UVJdXZ3OSaChoYHDhw8zaNCgLj34Qx+2tradTsD985//TGNjI++//36X707cT2fHfObMGU6fPv3ADxx5UL6+vmzevJl9+/ZpJztpNBr27t1Lnz59FP+919XVcfLkSaZNm2aUJxra29uTm5tLWVlZh6fRHjx4EFNTU6MN42h36NAhzp492+UndnbViy++yJAhQ9izZw+vv/66dkLo7t276dGjR4cTaHdx6NAh1qxZg7+/P9HR0UbLValUmJub60y63bt3L6C78pGhvPfee9qlF9udOnWKf/zjH7z33nuK3rhRqVRYWFh0GD7T3NzM1q1b6du3r2L9ytChQxk2bBjp6eksWLBAu8rVoUOHaGhoMMrKbXD75sWTTz6Jj4+P4lnt56WDBw92WFnp2LFjNDY24uDgoPg+3Kmuro4tW7Ywbty4B36wpj51h7m5OR4eHqSmpjJ//nztkMfU1FQaGxvx9fU1yHE8LqTQ7ybi4+MBtOvIpqam8p///AcLCwtCQ0MVyfzjH/9IZmYmkyZNQqVSkZqaqn2vb9++eHt7K5K7ZMkSevfujaurK9bW1ly+fJkDBw5QWVmpaDFkbm7e6TFt374dU1NTxY4Xbh/zk08+iaurKwMGDOC///0ve/bsYcCAASxatEixXLhdZMyYMYPExERqa2txcHDgyy+/JCcnhxUrVih+l/nQoUPcunXLaHdhIiIiyM7OJigoiJCQECwtLTl+/DjZ2dnMmjVLsQtJuD05LzExEU9PT/r3709+fj7Jycn4+/szbdo0g2bp02e8++67LFy4kIiICPz8/CgtLWXnzp0EBgZ2eaUnfXIzMzO1Q6TUajUlJSXarwsICNBZ794QuQUFBbz77rv0798fDw8P7bKH7Tw9Pbs8xOJ+2ZmZmSQkJODj48PgwYNpamoiJyeHnJwcJk6c2OXi8365nd1NbR+6MmbMmIf61EafY/7rX//K1KlTsbGxQaVSkZyczMWLF/n444+7PB9Fn7+v6OhoIiMjCQ4OJiAggOrqarZv346DgwO/+tWvFDvmdiqVihMnTjBlyhSDzLu5X+6kSZN48cUXiY2NpaKiAmdnZy5evMjOnTt55plneO211xTLhttzj9zc3Hj++eeprq5mz549tLW1sXr16gfO07fuWLp0KbNmzSIsLIw33niDyspKtm3bxvjx4xk7dmyXj/dxZKJ5VE++EQZ1r7uMNjY2HZYINKSwsDC++uoro+fu27eP1NRUzp8/z/Xr1zE3N9eug+zu7q5I5v8TFhbG9evXO3Q4hrZjxw7S09MpKyujoaEBKysrxo0bx6JFi3juuecUy22nVquJj48nJSWFmpoabG1tmTNnjlFW6ggMDKS8vJwTJ04ouszgnQoKCoiNjaWoqAiVSoWNjQ0zZ84kIiJC0X24ePEiq1evprCwkBs3bvDCCy/wxhtvEBoaavAJ7vr2GUePHiUuLo4LFy5gZWXFzJkziYqK6vLkTX1yo6OjSU5O7nS7HTt2MGbMGIPnHjhw4P9O5u9qrj7ZpaWlJCYm8s0331BTU0OPHj2wt7fH39+fsLAwnTXIDZXbmfafQ0pKykMV+vfLPnfuHHFxcRQWFlJXV0evXr1wdHQkPDycSZMmKZbbLjs7m9jYWEpKSujTpw9eXl4sX778oYZC6pudlJTEqlWrSEhIMMhwRH1yr127Rnx8PMePH+fHH3+kb9++eHp6smzZsi5dOD9I9po1a8jKyqKqqgpLS0smTJjA4sWLdeZB6eNB6o4zZ84QExNDYWEh/fr1w8/Pj2XLlj3UBODHkRT6QgghhBBCdEOy6o4QQgghhBDdkBT6QgghhBBCdENS6AshhBBCCNENSaEvhBBCCCFENySFvhBCCCGEEN2QFPpCCCGEEEJ0Q1LoCyGEEEII0Q1JoS+EEKJbCQsLM8iDiIQQ4nHXtUcbCiGE+Fk5ffo0v/3tb+/5vqmpKYWFhUbcIyGEEPcjhb4QQgi9TZ8+nfHjx+u09+ghHxALIcRPjRT6Qggh9Obg4EBAQMCj3g0hhBB6kFswQgghDKaiooLhw4cTGxtLRkYG/v7+jBw5kokTJxIbG8utW7d0vqa4uJi33nqLMWPGMHLkSPz8/Ni8eTOtra0621ZXV7NmzRq8vLxwcnLCw8ODuXPnkpubq7NtVVUVy5YtY/To0Tg7OxMREcEPP/ygyHELIcRPkdzRF0IIobempibq6up02nv16kW/fv20rzMzMykvLyckJISBAweSmZlJXFwcP/74I+vWrdNud/bsWcLCwujZs6d226ysLGJiYiguLmb9+vXabSsqKggKCqK2tpaAgACcnJxoamri22+/JS8vD09PT+22jY2NhIaG4uzszNKlS6moqGDHjh1ERUWRkZGBqampQj8hIYT46ZBCXwghhN5iY2OJjY3VaZ84cSKJiYna18XFxezbtw9HR0cAQkNDefvttzlw4ACBgYG4uLgAsHbtWtRqNUlJSYwYMUK77ZIlS8jIyOD111/Hw8MDgN///vdcuXKFLVu28Oqrr3bIb2tr6/D66tWrREREEBkZqW2zsrLiT3/6E3l5eTpfL4QQ3ZEU+kIIIfQWGBiIr6+vTruVlVWH12PHjtUW+QAmJia8+eabHD16lCNHjuDi4kJtbS3ffPMNPj4+2iK/fduFCxfy73//myNHjuDh4YFKpeLEiRO8+uqrnRbpd08G7tGjh84qQa+88goAly5dkkJfCPGzIIW+EEIIvT3//POMHTv2vtsNHTpUp+0Xv/gFAOXl5cDtoTh3tt9pyJAh9OjRQ7ttWVkZGo0GBwcHvfbz6aefpnfv3h3a+vfvD4BKpdLrewghxONOJuMKIYTodv7fGHyNRmPEPRFCiEdHCn0hhBAGd+HCBZ228+fPA2BnZweAra1th/Y7ff/997S1tWm3HTx4MCYmJhQVFSm1y0II0e1IoS+EEMLg8vLy+O6777SvNRoNW7ZsAcDb2xuAp556CldXV7KysigtLe2w7d/+9jcAfHx8gNvDbsaPH092djZ5eXk6eXKXXgghdMkYfSGEEHorLCwkNTW10/faC3iAESNGMHv2bEJCQrC2tubYsWPk5eUREBCAq6urdruVK1cSFhZGSEgIwcHBWFtbk5WVRU5ODtOnT9euuAPw4YcfUlhYSGRkJDNmzMDR0ZHm5ma+/fZbbGxsWLFihXIHLoQQjyEp9IUQQugtIyODjIyMTt/74osvtGPjJ0+ejL29PYmJifzwww889dRTREVFERUV1eFrRo4cSVJSEhs3bmT37t00NjZiZ2fH8uXLCQ8P77CtnZ0d+/fvZ9OmTWRnZ5OamoqFhQUjRowgMDBQmQMWQojHmIlGPu8UQghhIBUVFXh5efH222+zaNGiR707QgjxsyZj9IUQQgghhOiGpNAXQgghhBCiG5JCXwghhBBCiG5IxugLIYQQQgjRDckdfSGEEEIIIbohKfSFEEIIIYTohqTQF0IIIYQQohuSQl8IIYQQQohuSAp9IYQQQgghuiEp9IUQQgghhOiG/gcjMtl7BO4w0QAAAABJRU5ErkJggg==\n","text/plain":["<Figure size 864x432 with 1 Axes>"]},"metadata":{"tags":[]}}]},{"cell_type":"code","metadata":{"id":"DbiTDpVv3kiF","colab":{"base_uri":"https://localhost:8080/"},"outputId":"22407097-7585-46d1-ed2a-41957fa89cf5"},"source":["import os\n","\n","\n","output_dir = 'model_euclidean_USE_cos_QC/'\n","\n","# Create output directory if needed\n","if not os.path.exists(output_dir):\n","    os.makedirs(output_dir)\n","\n","print(\"Saving model to %s\" % output_dir)\n","\n","\n","# model_to_save = model.module if hasattr(model, 'module') else model  # Take care of distributed/parallel training\n","# model_to_save.save_pretrained(output_dir)\n","tokenizer.save_pretrained(output_dir)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Saving model to model_euclidean_USE_cos_QC/\n"],"name":"stdout"},{"output_type":"execute_result","data":{"text/plain":["('model_euclidean_USE_cos_QC/vocab.txt',\n"," 'model_euclidean_USE_cos_QC/special_tokens_map.json',\n"," 'model_euclidean_USE_cos_QC/added_tokens.json')"]},"metadata":{"tags":[]},"execution_count":70}]},{"cell_type":"code","metadata":{"id":"1U2UQ29a3kiI"},"source":["# !pip install joblib\n","# import joblib\n","# joblib.dump(LE, \"label_encoder\")\n"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"0F5PxZm9vAOI"},"source":["import json\n","torch.save(model.state_dict(), os.path.join(output_dir, 'model_weights'))\n"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Vlb0IpVJO1XQ"},"source":["# with open(os.path.join(output_dir, 'model_config.json'), 'w') as f:\n","#     json.dump(model.config, f)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"dpGY8vSDI6u4","colab":{"base_uri":"https://localhost:8080/"},"outputId":"81983680-2217-4cf4-c219-74ee7e72c6f7"},"source":["!zip -r model_euclidean_USE_cos_QC.zip model_euclidean_USE_cos_QC\n","# files.download('model_euclidean_1.zip')"],"execution_count":null,"outputs":[{"output_type":"stream","text":["  adding: model_euclidean_USE_cos_QC/ (stored 0%)\n","  adding: model_euclidean_USE_cos_QC/tokenizer_config.json (stored 0%)\n","  adding: model_euclidean_USE_cos_QC/vocab.txt (deflated 53%)\n","  adding: model_euclidean_USE_cos_QC/model_weights (deflated 7%)\n","  adding: model_euclidean_USE_cos_QC/special_tokens_map.json (deflated 40%)\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"HvFDCDIxKDOf"},"source":["# !zip -r label_encoder_categorized_reduced.zip label_encoder\n","# files.download('label_encoder_categorized_reduced.zip')"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"ijMg4mgia6Xz"},"source":["# Inference"]},{"cell_type":"code","metadata":{"id":"4178_yLFMWmx"},"source":["test_features = test_features.values\n","labels = test_labels.values"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"3ZpmBJuIC2nM","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625202805337,"user_tz":-330,"elapsed":8,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"5ef2e89b-13e8-435d-d910-7287faed7345"},"source":["test_features"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["array([\"Robert is a fisherman who wants to find a way to catch more fish. He decided to try different sizes of hooks. Robert caught 4 catfish, 3 trout, and 7 perch while using worms for bait. Which is the independent (manipulated) variable in Robert's investigation? (A) type of bait (B) size of hook (C) type of fish caught (D) number of fish caught\",\n","       'Which of these factors causes water to evaporate the fastest? (A) high temperatures (B) high humidity (C) slow winds (D) slow runoff',\n","       'Which statement is true about the particles of a liquid compared to the particles of a gas? (A) Particles of a liquid are a slower and further apart. (B) Particles of a liquid are faster and farther apart. (C) Particles of a liquid are slower and closer together. (D) Particles of a liquid are faster and closer together.',\n","       ...,\n","       'Which of these is not an inherited trait in humans? (A) height (B) hair color (C) skin color (D) intelligence',\n","       'In order to survive, all animals need (A) heat, water, and soil (B) sunlight, soil, and heat (C) sunlight, air, and food (D) food, water, and air',\n","       'Scientists claim that the continents of South America and Africa were once a single landmass. All of the following observations support this claim except (A) the mountains on these continents have similar rocks of the same age. (B) these continents appear to fit together like the pieces of a puzzle. (C) similar fish live in the ocean off the coasts of these continents. (D) the same kinds of fossils have been found on these continents.'],\n","      dtype=object)"]},"metadata":{"tags":[]},"execution_count":47}]},{"cell_type":"code","metadata":{"id":"MRZ54gFokNh9","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625202808433,"user_tz":-330,"elapsed":341,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"19cb90a5-6352-4d84-b500-341468c72a52"},"source":["labels"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["array(['science_INFERENCE_experiment design',\n","       'matter_Change of state_EVAPoration', 'matter_chemistry_atomic',\n","       ..., 'Life_reproduction_DNA inheritance_inheritance',\n","       'Life_functions_FUNCT_animalESS', 'EARTH_INNER_PLATE_CONTDRIFT'],\n","      dtype=object)"]},"metadata":{"tags":[]},"execution_count":48}]},{"cell_type":"code","metadata":{"id":"8ohj1x7frQJ1","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625202810557,"user_tz":-330,"elapsed":417,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"aa235f1d-e89a-48e1-9875-d9d9084d8cf7"},"source":["len(list(set(labels)))"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["352"]},"metadata":{"tags":[]},"execution_count":49}]},{"cell_type":"code","metadata":{"id":"r-lbmV5JiRCz"},"source":["import pandas as pd\n","targets = pd.read_csv(\"targets_ARC.csv\")\n","targets = targets[\"targets\"].values"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Kjinn0gXkNuP"},"source":["\n","# course_taxonomy\n","targets = list(set(targets))\n","poincare_emb_data_test = get_cleaned_taxonomy(targets)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"6wl0RJ3SSW4i","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625203841548,"user_tz":-330,"elapsed":3066,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"85846d19-d7b1-4901-cd22-2ffa867f8dbd"},"source":["# taxonomy_vectors = []\n","taxonomy_vectors = use_embedding.get_tokenized_sents_embeddings_USE(poincare_emb_data_test)\n","# taxonomy_vectors = np.vstack(taxonomy_vectors)\n","taxonomy_vectors.shape"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["(352, 512)"]},"metadata":{"tags":[]},"execution_count":168}]},{"cell_type":"code","metadata":{"id":"Nso39n1N_po_","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625203844624,"user_tz":-330,"elapsed":3079,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"4d07d6a7-4140-4c3c-822b-b5206782d736"},"source":["model = MulticlassClassifier('bert-base-uncased')\n","model.load_state_dict(torch.load('model_euclidean_USE_cos_QC/model_weights'))\n","model.cuda()"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["MulticlassClassifier(\n","  (bert): BertModel(\n","    (embeddings): BertEmbeddings(\n","      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n","      (position_embeddings): Embedding(512, 768)\n","      (token_type_embeddings): Embedding(2, 768)\n","      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","      (dropout): Dropout(p=0.1, inplace=False)\n","    )\n","    (encoder): BertEncoder(\n","      (layer): ModuleList(\n","        (0): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=768, out_features=768, bias=True)\n","              (key): Linear(in_features=768, out_features=768, bias=True)\n","              (value): Linear(in_features=768, out_features=768, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=768, out_features=768, bias=True)\n","              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=768, out_features=3072, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=3072, out_features=768, bias=True)\n","            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (1): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=768, out_features=768, bias=True)\n","              (key): Linear(in_features=768, out_features=768, bias=True)\n","              (value): Linear(in_features=768, out_features=768, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=768, out_features=768, bias=True)\n","              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=768, out_features=3072, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=3072, out_features=768, bias=True)\n","            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (2): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=768, out_features=768, bias=True)\n","              (key): Linear(in_features=768, out_features=768, bias=True)\n","              (value): Linear(in_features=768, out_features=768, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=768, out_features=768, bias=True)\n","              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=768, out_features=3072, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=3072, out_features=768, bias=True)\n","            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (3): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=768, out_features=768, bias=True)\n","              (key): Linear(in_features=768, out_features=768, bias=True)\n","              (value): Linear(in_features=768, out_features=768, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=768, out_features=768, bias=True)\n","              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=768, out_features=3072, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=3072, out_features=768, bias=True)\n","            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (4): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=768, out_features=768, bias=True)\n","              (key): Linear(in_features=768, out_features=768, bias=True)\n","              (value): Linear(in_features=768, out_features=768, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=768, out_features=768, bias=True)\n","              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=768, out_features=3072, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=3072, out_features=768, bias=True)\n","            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (5): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=768, out_features=768, bias=True)\n","              (key): Linear(in_features=768, out_features=768, bias=True)\n","              (value): Linear(in_features=768, out_features=768, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=768, out_features=768, bias=True)\n","              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=768, out_features=3072, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=3072, out_features=768, bias=True)\n","            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (6): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=768, out_features=768, bias=True)\n","              (key): Linear(in_features=768, out_features=768, bias=True)\n","              (value): Linear(in_features=768, out_features=768, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=768, out_features=768, bias=True)\n","              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=768, out_features=3072, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=3072, out_features=768, bias=True)\n","            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (7): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=768, out_features=768, bias=True)\n","              (key): Linear(in_features=768, out_features=768, bias=True)\n","              (value): Linear(in_features=768, out_features=768, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=768, out_features=768, bias=True)\n","              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=768, out_features=3072, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=3072, out_features=768, bias=True)\n","            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (8): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=768, out_features=768, bias=True)\n","              (key): Linear(in_features=768, out_features=768, bias=True)\n","              (value): Linear(in_features=768, out_features=768, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=768, out_features=768, bias=True)\n","              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=768, out_features=3072, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=3072, out_features=768, bias=True)\n","            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (9): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=768, out_features=768, bias=True)\n","              (key): Linear(in_features=768, out_features=768, bias=True)\n","              (value): Linear(in_features=768, out_features=768, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=768, out_features=768, bias=True)\n","              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=768, out_features=3072, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=3072, out_features=768, bias=True)\n","            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (10): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=768, out_features=768, bias=True)\n","              (key): Linear(in_features=768, out_features=768, bias=True)\n","              (value): Linear(in_features=768, out_features=768, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=768, out_features=768, bias=True)\n","              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=768, out_features=3072, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=3072, out_features=768, bias=True)\n","            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (11): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=768, out_features=768, bias=True)\n","              (key): Linear(in_features=768, out_features=768, bias=True)\n","              (value): Linear(in_features=768, out_features=768, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=768, out_features=768, bias=True)\n","              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=768, out_features=3072, bias=True)\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=3072, out_features=768, bias=True)\n","            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","      )\n","    )\n","    (pooler): BertPooler(\n","      (dense): Linear(in_features=768, out_features=768, bias=True)\n","      (activation): Tanh()\n","    )\n","  )\n","  (dropout): Dropout(p=0.1, inplace=False)\n","  (fc1): Linear(in_features=768, out_features=384, bias=True)\n","  (fc2): Linear(in_features=384, out_features=512, bias=True)\n",")"]},"metadata":{"tags":[]},"execution_count":169}]},{"cell_type":"code","metadata":{"id":"Qe4qYkV2C4fX"},"source":["test_input_ids = []\n","test_attention_masks = []\n","for sent in test_features:\n","\n","    encoded_dict = tokenizer.encode_plus(\n","                        sent,                      # Sentence to encode.\n","                        add_special_tokens = True, # Add '[CLS]' and '[SEP]'\n","                        max_length = 128,           # Pad & truncate all sentences.\n","                        pad_to_max_length = True,\n","                        truncation=True,\n","                        return_attention_mask = True,   # Construct attn. masks.\n","                        return_tensors = 'pt',     # Return pytorch tensors.\n","                   )\n","    \n","    # Add the encoded sentence to the list.    \n","    test_input_ids.append(encoded_dict['input_ids'])\n","    \n","    # And its attention mask (simply differentiates padding from non-padding).\n","    test_attention_masks.append(encoded_dict['attention_mask'])\n","\n","# Convert the lists into tensors.\n","test_input_ids = torch.cat(test_input_ids, dim=0)\n","test_attention_masks = torch.cat(test_attention_masks, dim=0)\n","# labels = torch.tensor(labels)\n","\n","# Set the batch size.  \n","batch_size = 32  \n","test_poincare_tensor = torch.tensor(taxonomy_vectors,dtype=torch.float)\n","\n","# Create the DataLoader.\n","# prediction_data = TensorDataset(test_input_ids, test_attention_masks, test_poincare_tensor)\n","# prediction_sampler = SequentialSampler(prediction_data)\n","# prediction_dataloader = DataLoader(prediction_data, sampler=prediction_sampler, batch_size=batch_size)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"SNdlve8AJcCO"},"source":["from torch.utils.data import DataLoader, RandomSampler, SequentialSampler\n","test_poincare_tensor = torch.tensor(taxonomy_vectors,dtype=torch.float)\n"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"I6aMBHkAQZjT"},"source":["cos = torch.nn.CosineSimilarity(dim=0, eps=1e-6)\n","def dist_without_grad( u, v):\n","  sqdist = torch.sum((u - v) ** 2, dim=-1)\n","  squnorm = torch.sum(u ** 2, dim=-1)\n","  sqvnorm = torch.sum(v ** 2, dim=-1)\n","  x = 1 + 2 * sqdist / ((1 - squnorm) * (1 - sqvnorm)) + 1e-7\n","  z = torch.sqrt(x ** 2 - 1)\n","  return torch.log(x + z)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"idBN7kS5ebZ1","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625203846131,"user_tz":-330,"elapsed":35,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"184b7a57-87f3-459d-bbc9-8b77ae96a94e"},"source":["len(labels)"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["352"]},"metadata":{"tags":[]},"execution_count":173}]},{"cell_type":"code","metadata":{"id":"oe0otXOPg7z0"},"source":["targets = np.array(targets)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"s8sPJhJJk_gQ"},"source":["def precision(actual, predicted, k):\n","    act_set = set(actual)\n","    pred_set = set(predicted[:k])\n","    result = len(act_set & pred_set) / float(k)\n","    return result\n","\n","def recall(actual, predicted, k):\n","    act_set = set(actual)\n","    pred_set = set(predicted[:k])\n","    result = len(act_set & pred_set) / float(len(act_set))\n","    return result"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"z6wOPMWKrLqG"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"tA-1RvWKfV7C","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625203846134,"user_tz":-330,"elapsed":25,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"5dc31467-dfac-4015-9255-486adf5e8379"},"source":["test_poincare_tensor.shape"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["torch.Size([352, 512])"]},"metadata":{"tags":[]},"execution_count":176}]},{"cell_type":"code","metadata":{"id":"mPCktQT9DVT4","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625203868818,"user_tz":-330,"elapsed":22702,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"9c8bdf21-8199-437f-b6a5-3d39b6019fbd"},"source":["# Prediction on test set\n","\n","print('Predicting labels for {:,} test sentences...'.format(len(test_input_ids)))\n","\n","# Put model in evaluation mode\n","model.eval()\n","cos = torch.nn.CosineSimilarity(dim=1, eps=1e-6)\n","\n","input_ids = test_input_ids.to('cuda')\n","attention_masks = test_attention_masks.to('cuda')\n","test_poincare_tensor = test_poincare_tensor.to('cuda')\n","# Tracking variables \n","predictions , true_labels = [], []\n","for input_id,attention_mask in zip(input_ids, attention_masks):\n","  with torch.no_grad():\n","    outputs = model(input_id.reshape(1,-1),attention_mask.reshape(1,-1))\n","    \n","  distances = cos(outputs,test_poincare_tensor)\n","  distances,indices = torch.topk(distances,10,largest=True)\n","  predictions.append(targets[indices.cpu().numpy()])\n","print(len(predictions))\n","\n","print('    DONE.')\n","# predictions"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Predicting labels for 1,400 test sentences...\n","1400\n","    DONE.\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"BNKgkK7-fCcQ","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625203868819,"user_tz":-330,"elapsed":42,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"cc082ec8-c3f1-40e2-e899-bc9c9a15d86c"},"source":["labels\n","\n","\n"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["array([107, 168,  77, 183, 281, 254, 142, 249,  81, 103, 312, 299, 171,\n","       337, 179, 208, 285, 155, 335, 342, 137, 243, 188,  69, 217, 273,\n","        16, 344,  88, 316, 132,  85, 118,  20, 244, 125, 348, 332, 164,\n","       280,  33, 256, 251, 100, 216, 278, 279, 287,   3, 204,  71, 269,\n","       334, 196,  72, 158,   7, 172, 173, 325, 297,  11, 295, 195, 180,\n","        89, 308,  48, 274,  43, 264, 139, 104, 230,  95, 241, 154, 231,\n","        37,  87, 229, 145,  75, 190, 286, 234, 101,  51, 199,  68, 283,\n","        29, 112, 326, 184, 296, 162, 189, 343, 303,  82, 201, 122,  73,\n","       108, 141,   6,  14, 257,  97, 330, 111,  39, 175, 213, 300,  67,\n","       311, 255, 152, 338, 349,  46, 130, 347, 177, 176, 320, 246, 220,\n","         1, 110, 121, 271, 277, 282,  53, 310, 322, 319, 307, 170,  98,\n","       167, 236, 309, 346, 131,  25, 209, 315,  49, 293,  52, 259, 106,\n","       181, 268, 267, 197, 126,  38,  34, 317,  83, 219, 266, 265, 245,\n","        94, 305, 324, 124, 105,  22,   2, 214, 146, 260,  80, 138, 238,\n","       331, 123, 233, 345, 192, 228, 115, 207, 288, 341, 144, 247, 242,\n","       148, 151,  64, 327, 136,  10,  78, 150, 329, 163,  21, 153, 147,\n","       321, 198,  63, 248,   9,  91, 222, 211,  79,  26, 302,  90, 340,\n","       140,  18,  96, 294, 127,  30,  66, 284,  13,  36, 262, 159,  62,\n","        60,  19, 350, 252,  28,   5, 232, 298, 275, 304, 215, 119,  99,\n","       133, 117, 351,  15,  44, 221, 250, 272,  86,   8, 339, 191,  12,\n","       227, 240,  92, 193, 174,  24,  41, 235, 289, 314, 333, 291, 182,\n","       313, 186,  57,  70,  59, 113, 160, 161, 128,  76, 178, 149, 135,\n","       116, 224,   4, 166, 102, 276,  27,  23, 318,  56,  84,  50, 187,\n","       200, 237, 143,  35, 239, 203, 223, 206, 114,  47, 323,  45, 185,\n","       134,  17,  65, 328, 253, 205, 292, 263,  40, 109, 258, 225,  93,\n","       157, 301, 212, 120,  74, 336,  55, 306, 270, 165, 290, 202,   0,\n","       194, 210, 226,  58, 261,  54,  61, 218,  31,  32,  42, 169, 156,\n","       129])"]},"metadata":{"tags":[]},"execution_count":178}]},{"cell_type":"code","metadata":{"id":"O5wj2gXYFkrQ"},"source":["# labels=train['QCLabel'].values"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Y_H7xGlwp-is","executionInfo":{"status":"ok","timestamp":1625203868820,"user_tz":-330,"elapsed":38,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"9a94c718-845c-49dc-a5e0-64172425b4fe"},"source":["test['QCLabel'].values"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["array(['science_INFERENCE_experiment design',\n","       'matter_Change of state_EVAPoration', 'matter_chemistry_atomic',\n","       ..., 'Life_reproduction_DNA inheritance_inheritance',\n","       'Life_functions_FUNCT_animalESS', 'EARTH_INNER_PLATE_CONTDRIFT'],\n","      dtype=object)"]},"metadata":{"tags":[]},"execution_count":180}]},{"cell_type":"code","metadata":{"id":"KmvLFX-E0C2i"},"source":["targets = pd.read_csv(\"targets_ARC.csv\")\n","\n","labels = targets[\"targets\"].values"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"nOt8hvs-CZfn","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625203868821,"user_tz":-330,"elapsed":34,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"b60658de-0d04-4c0a-800c-77ac65ca790f"},"source":["from sklearn .preprocessing import LabelEncoder\n","LE= LabelEncoder()\n","labels = LE.fit_transform(labels)\n","labels"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["array([107, 168,  77, 183, 281, 254, 142, 249,  81, 103, 312, 299, 171,\n","       337, 179, 208, 285, 155, 335, 342, 137, 243, 188,  69, 217, 273,\n","        16, 344,  88, 316, 132,  85, 118,  20, 244, 125, 348, 332, 164,\n","       280,  33, 256, 251, 100, 216, 278, 279, 287,   3, 204,  71, 269,\n","       334, 196,  72, 158,   7, 172, 173, 325, 297,  11, 295, 195, 180,\n","        89, 308,  48, 274,  43, 264, 139, 104, 230,  95, 241, 154, 231,\n","        37,  87, 229, 145,  75, 190, 286, 234, 101,  51, 199,  68, 283,\n","        29, 112, 326, 184, 296, 162, 189, 343, 303,  82, 201, 122,  73,\n","       108, 141,   6,  14, 257,  97, 330, 111,  39, 175, 213, 300,  67,\n","       311, 255, 152, 338, 349,  46, 130, 347, 177, 176, 320, 246, 220,\n","         1, 110, 121, 271, 277, 282,  53, 310, 322, 319, 307, 170,  98,\n","       167, 236, 309, 346, 131,  25, 209, 315,  49, 293,  52, 259, 106,\n","       181, 268, 267, 197, 126,  38,  34, 317,  83, 219, 266, 265, 245,\n","        94, 305, 324, 124, 105,  22,   2, 214, 146, 260,  80, 138, 238,\n","       331, 123, 233, 345, 192, 228, 115, 207, 288, 341, 144, 247, 242,\n","       148, 151,  64, 327, 136,  10,  78, 150, 329, 163,  21, 153, 147,\n","       321, 198,  63, 248,   9,  91, 222, 211,  79,  26, 302,  90, 340,\n","       140,  18,  96, 294, 127,  30,  66, 284,  13,  36, 262, 159,  62,\n","        60,  19, 350, 252,  28,   5, 232, 298, 275, 304, 215, 119,  99,\n","       133, 117, 351,  15,  44, 221, 250, 272,  86,   8, 339, 191,  12,\n","       227, 240,  92, 193, 174,  24,  41, 235, 289, 314, 333, 291, 182,\n","       313, 186,  57,  70,  59, 113, 160, 161, 128,  76, 178, 149, 135,\n","       116, 224,   4, 166, 102, 276,  27,  23, 318,  56,  84,  50, 187,\n","       200, 237, 143,  35, 239, 203, 223, 206, 114,  47, 323,  45, 185,\n","       134,  17,  65, 328, 253, 205, 292, 263,  40, 109, 258, 225,  93,\n","       157, 301, 212, 120,  74, 336,  55, 306, 270, 165, 290, 202,   0,\n","       194, 210, 226,  58, 261,  54,  61, 218,  31,  32,  42, 169, 156,\n","       129])"]},"metadata":{"tags":[]},"execution_count":182}]},{"cell_type":"code","metadata":{"id":"adb4gTNgGKUP","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625203868821,"user_tz":-330,"elapsed":31,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"29c90e11-663a-4dcb-c9c5-20a835e34aa0"},"source":["labels"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["array([107, 168,  77, 183, 281, 254, 142, 249,  81, 103, 312, 299, 171,\n","       337, 179, 208, 285, 155, 335, 342, 137, 243, 188,  69, 217, 273,\n","        16, 344,  88, 316, 132,  85, 118,  20, 244, 125, 348, 332, 164,\n","       280,  33, 256, 251, 100, 216, 278, 279, 287,   3, 204,  71, 269,\n","       334, 196,  72, 158,   7, 172, 173, 325, 297,  11, 295, 195, 180,\n","        89, 308,  48, 274,  43, 264, 139, 104, 230,  95, 241, 154, 231,\n","        37,  87, 229, 145,  75, 190, 286, 234, 101,  51, 199,  68, 283,\n","        29, 112, 326, 184, 296, 162, 189, 343, 303,  82, 201, 122,  73,\n","       108, 141,   6,  14, 257,  97, 330, 111,  39, 175, 213, 300,  67,\n","       311, 255, 152, 338, 349,  46, 130, 347, 177, 176, 320, 246, 220,\n","         1, 110, 121, 271, 277, 282,  53, 310, 322, 319, 307, 170,  98,\n","       167, 236, 309, 346, 131,  25, 209, 315,  49, 293,  52, 259, 106,\n","       181, 268, 267, 197, 126,  38,  34, 317,  83, 219, 266, 265, 245,\n","        94, 305, 324, 124, 105,  22,   2, 214, 146, 260,  80, 138, 238,\n","       331, 123, 233, 345, 192, 228, 115, 207, 288, 341, 144, 247, 242,\n","       148, 151,  64, 327, 136,  10,  78, 150, 329, 163,  21, 153, 147,\n","       321, 198,  63, 248,   9,  91, 222, 211,  79,  26, 302,  90, 340,\n","       140,  18,  96, 294, 127,  30,  66, 284,  13,  36, 262, 159,  62,\n","        60,  19, 350, 252,  28,   5, 232, 298, 275, 304, 215, 119,  99,\n","       133, 117, 351,  15,  44, 221, 250, 272,  86,   8, 339, 191,  12,\n","       227, 240,  92, 193, 174,  24,  41, 235, 289, 314, 333, 291, 182,\n","       313, 186,  57,  70,  59, 113, 160, 161, 128,  76, 178, 149, 135,\n","       116, 224,   4, 166, 102, 276,  27,  23, 318,  56,  84,  50, 187,\n","       200, 237, 143,  35, 239, 203, 223, 206, 114,  47, 323,  45, 185,\n","       134,  17,  65, 328, 253, 205, 292, 263,  40, 109, 258, 225,  93,\n","       157, 301, 212, 120,  74, 336,  55, 306, 270, 165, 290, 202,   0,\n","       194, 210, 226,  58, 261,  54,  61, 218,  31,  32,  42, 169, 156,\n","       129])"]},"metadata":{"tags":[]},"execution_count":183}]},{"cell_type":"code","metadata":{"id":"UsartdcbsLFz"},"source":["test_labels = LE.transform(test[\"QCLabel\"].values)"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"OYqa-JG6o7gL"},"source":["# To test generalization\n","Skip 3 cells not copulsory for inference"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"RZIY_54gHm7M","executionInfo":{"status":"ok","timestamp":1625203868822,"user_tz":-330,"elapsed":28,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"5efcbb6e-1705-40cb-97c0-9168dd030bef"},"source":["np.where(labels==325)"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["(array([59]),)"]},"metadata":{"tags":[]},"execution_count":185}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"O96135r8GhD6","executionInfo":{"status":"ok","timestamp":1625203868822,"user_tz":-330,"elapsed":24,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"fa85d31c-29ed-4749-c957-f1d0fba49db2"},"source":["LE.transform([\"matter_properties of material_REFLECT\"])"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["array([325])"]},"metadata":{"tags":[]},"execution_count":186}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"lTZJoePzIQ1e","executionInfo":{"status":"ok","timestamp":1625203868823,"user_tz":-330,"elapsed":21,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"d865ab8d-06b8-4de5-8532-74889eb7843f"},"source":["#unseen label proof\n","\n","np.where(test[\"QCLabel\"].values==\"matter_properties of material_REFLECT\")"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["(array([ 498,  644, 1102]),)"]},"metadata":{"tags":[]},"execution_count":187}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Pq_ukFUBHW6q","executionInfo":{"status":"ok","timestamp":1625203868823,"user_tz":-330,"elapsed":19,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"1d815233-6ad0-46b7-ed1c-acf9f64a1f38"},"source":["#unseen label proof\n","\n","LE.inverse_transform([325])"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["array(['matter_properties of material_REFLECT'], dtype=object)"]},"metadata":{"tags":[]},"execution_count":188}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"t1bcG5-w5pud","executionInfo":{"status":"ok","timestamp":1625203868824,"user_tz":-330,"elapsed":17,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"211ba0e5-7e15-467c-c4b4-61c9fa2e5559"},"source":["LE.inverse_transform([320])"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["array(['matter_properties of material_FLEX'], dtype=object)"]},"metadata":{"tags":[]},"execution_count":189}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"GwH2zIAW8QR_","executionInfo":{"status":"ok","timestamp":1625203868824,"user_tz":-330,"elapsed":14,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"d6028035-5339-4339-c320-99c5bb29fb87"},"source":["LE.inverse_transform([246])"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["array(['energy_LIGHT_REFLECT'], dtype=object)"]},"metadata":{"tags":[]},"execution_count":190}]},{"cell_type":"markdown","metadata":{"id":"CbohQzAhlYRN"},"source":[""]},{"cell_type":"code","metadata":{"id":"azkCfK-bCoXd"},"source":["final_predictions = []\n","for prediction in predictions:\n","  final_predictions.append(LE.transform(prediction))\n"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"fF7tvRJcqe9H","executionInfo":{"status":"ok","timestamp":1625203869361,"user_tz":-330,"elapsed":12,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"a6f062d1-cf93-4d3f-f53a-de1e98fed15c"},"source":["predictions[1102]"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["array(['matter_properties of material_FLEX',\n","       'matter_properties of material_REFLECT',\n","       'matter_properties of objects', 'matter_properties of material',\n","       'matter_properties of material_HARD',\n","       'matter_properties of material_CLASS',\n","       'matter_properties of material_SPECIFICHEAT',\n","       'matter_properties of material_LUSTER',\n","       'matter_properties of material_ELECCOND',\n","       'matter_properties of material_DENSITY'], dtype='<U73')"]},"metadata":{"tags":[]},"execution_count":192}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"buXWtwdNHSYF","executionInfo":{"status":"ok","timestamp":1625204841574,"user_tz":-330,"elapsed":432,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"558705d1-573e-4104-a2e5-9109df19f17f"},"source":["#unseen label proof\n","final_predictions[1102]\n"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["array([320, 325, 329, 316, 321, 317, 327, 322, 319, 318])"]},"metadata":{"tags":[]},"execution_count":193}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"dm6Zd6Q05cOO","executionInfo":{"status":"ok","timestamp":1625204843421,"user_tz":-330,"elapsed":11,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"fa842c50-197e-4e98-e3ea-2867626b6c84"},"source":["final_predictions[498]\n"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["array([329, 280, 297, 317, 275, 347, 294, 296, 341, 340])"]},"metadata":{"tags":[]},"execution_count":194}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"exJ_nGI95fPl","executionInfo":{"status":"ok","timestamp":1625204844665,"user_tz":-330,"elapsed":8,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"888da89e-eed7-4624-f442-3139acc47b8f"},"source":["# Proof that Tagrec is able to adapt to changes in label space 325 () is not seen at train time\n","final_predictions[644]\n"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["array([246, 325, 247, 243, 245, 263, 244, 264, 328, 265])"]},"metadata":{"tags":[]},"execution_count":195}]},{"cell_type":"code","metadata":{"id":"OQrlczKxMwzZ","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1625204848838,"user_tz":-330,"elapsed":520,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"a76ba12d-a9ab-4476-cacc-73c879fd1f70"},"source":["len(final_predictions[-1])"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["10"]},"metadata":{"tags":[]},"execution_count":196}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":54},"id":"h2Yo8TNBxScC","executionInfo":{"status":"ok","timestamp":1625204849429,"user_tz":-330,"elapsed":12,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"84982cf9-9afa-4632-dc49-588ce89a0e2f"},"source":["test.loc[1102,\"Question\"]"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"application/vnd.google.colaboratory.intrinsic+json":{"type":"string"},"text/plain":["'A boy can see his face when he looks into a calm pond. Which physical property of the pond makes this happen? (A) flexibility (B) reflectiveness (C) temperature (D) volume'"]},"metadata":{"tags":[]},"execution_count":197}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":35},"id":"xjieMNsa5i-V","executionInfo":{"status":"ok","timestamp":1625204853153,"user_tz":-330,"elapsed":673,"user":{"displayName":"anonymous anonymous","photoUrl":"","userId":"06251276198200574004"}},"outputId":"8f6a9007-d443-4f12-867c-f4f387dc7cd1"},"source":["test.loc[644,\"Question\"]"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"application/vnd.google.colaboratory.intrinsic+json":{"type":"string"},"text/plain":["'Which object best reflects light? (A) gray door (B) white floor (C) black sweater (D) brown carpet'"]},"metadata":{"tags":[]},"execution_count":198}]},{"cell_type":"code","metadata":{"id":"Is-KTAENfB6C","colab":{"base_uri":"https://localhost:8080/"},"outputId":"511d967a-ff4a-4bbd-de36-7f4c31ab2f9a"},"source":["import tensorflow as tf\n","y_true = np.array(test_labels)\n","y_true = tf.identity(y_true)\n","y_pred = np.array(final_predictions)\n","y_pred = tf.identity(y_pred)\n","print(y_pred.shape,y_true.shape)\n","k = 20\n","recall, update_recall = tf.compat.v1.metrics.recall_at_top_k(y_true, y_pred, k=20)\n","precision, update_precision = tf.compat.v1.metrics.precision_at_top_k(y_true, y_pred, k=20)\n","\n","tmp_rank = tf.nn.top_k(y_pred, 20)\n","stream_vars = [i for i in tf.local_variables()]\n","\n","with tf.Session() as sess:\n","    sess.run(tf.local_variables_initializer())\n","    # print(\"precision\",sess.run(update_precision))\n","    # print(\"precision\",s|ess.run(precision))\n","\n","    print(\"update_recall: \",sess.run(update_recall ))\n","    print(\"recall\",sess.run(recall))\n","\n","    print(\"STREAM_VARS: \",(sess.run(stream_vars)))\n","    print(\"TMP_RANK: \",sess.run(y_pred))"],"execution_count":null,"outputs":[{"output_type":"stream","text":["(1400, 20) (1400,)\n","update_recall:  0.8878571428571429\n","recall 0.8878571428571429\n","STREAM_VARS:  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1243.0, 157.0, 0.0, 0.0]\n","TMP_RANK:  [[349 341 340 ...  87 104 102]\n"," [ 47  45  44 ...  50 279  39]\n"," [297  84 268 ... 320 269 283]\n"," ...\n"," [194 188 189 ... 175 107  87]\n"," [137 136 138 ... 129 127 161]\n"," [  7   9   6 ...   1  18  15]]\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"kI1jhndp6cEW","colab":{"base_uri":"https://localhost:8080/"},"outputId":"72e237d4-4df0-4a35-a1a4-aafabde25111"},"source":["import tensorflow as tf\n","y_true = np.array(test_labels)\n","y_true = tf.identity(y_true)\n","y_pred = np.array(final_predictions)\n","y_pred = tf.identity(y_pred)\n","print(y_pred.shape,y_true.shape)\n","k = 8\n","recall, update_recall = tf.compat.v1.metrics.recall_at_top_k(y_true, y_pred, 15)\n","precision, update_precision = tf.compat.v1.metrics.precision_at_top_k(y_true, y_pred, 15)\n","\n","tmp_rank = tf.nn.top_k(y_pred, 15)\n","stream_vars = [i for i in tf.local_variables()]\n","\n","with tf.Session() as sess:\n","    sess.run(tf.local_variables_initializer())\n","    print(\"precision\",sess.run(update_precision))\n","    # print(\"precision\",sess.run(precision))\n","\n","    print(\"update_recall: \",sess.run(update_recall ))\n","    print(\"recall\",sess.run(recall))\n","\n","    print(\"STREAM_VARS: \",(sess.run(stream_vars)))\n","    print(\"TMP_RANK: \",sess.run(y_pred))"],"execution_count":null,"outputs":[{"output_type":"stream","text":["(1400, 15) (1400,)\n","precision 0.05723809523809524\n","update_recall:  0.8585714285714285\n","recall 0.8585714285714285\n","STREAM_VARS:  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1202.0, 198.0, 1202.0, 19798.0]\n","TMP_RANK:  [[349 341 340 ... 343 345 348]\n"," [ 47  45  44 ... 117  76  40]\n"," [297  84 268 ... 330 266 331]\n"," ...\n"," [194 188 189 ...  89 127 128]\n"," [137 136 138 ...  78  95  96]\n"," [  7   9   6 ... 341 200 199]]\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"EaA9z5n3mZz0","colab":{"base_uri":"https://localhost:8080/"},"outputId":"da77f3df-7092-47f7-a623-c9c21917d093"},"source":["import tensorflow as tf\n","y_true = np.array(test_labels)\n","y_true = tf.identity(y_true)\n","y_pred = np.array(final_predictions)\n","y_pred = tf.identity(y_pred)\n","print(y_pred.shape,y_true.shape)\n","k = 8\n","recall, update_recall = tf.compat.v1.metrics.recall_at_top_k(y_true, y_pred, 10)\n","precision, update_precision = tf.compat.v1.metrics.precision_at_top_k(y_true, y_pred, 10)\n","\n","tmp_rank = tf.nn.top_k(y_pred, 10)\n","stream_vars = [i for i in tf.local_variables()]\n","\n","with tf.Session() as sess:\n","    sess.run(tf.local_variables_initializer())\n","    print(\"precision\",sess.run(update_precision))\n","    # print(\"precision\",sess.run(precision))\n","\n","    print(\"update_recall: \",sess.run(update_recall ))\n","    print(\"recall\",sess.run(recall))\n","\n","    print(\"STREAM_VARS: \",(sess.run(stream_vars)))\n","    print(\"TMP_RANK: \",sess.run(tmp_rank))"],"execution_count":null,"outputs":[{"output_type":"stream","text":["(1400, 10) (1400,)\n","precision 0.08085714285714286\n","update_recall:  0.8085714285714286\n","recall 0.8085714285714286\n","STREAM_VARS:  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1132.0, 268.0, 1132.0, 12868.0]\n","TMP_RANK:  TopKV2(values=array([[350, 349, 347, ..., 340, 337, 336],\n","       [282, 281,  65, ...,  45,  44,  31],\n","       [329, 297, 295, ...,  85,  84,  83],\n","       ...,\n","       [195, 194, 193, ..., 188, 187,  91],\n","       [138, 137, 136, ..., 118,  92,  90],\n","       [347, 346,  87, ...,   7,   6,   2]]), indices=array([[3, 0, 8, ..., 2, 6, 7],\n","       [4, 6, 8, ..., 1, 2, 7],\n","       [9, 0, 8, ..., 6, 1, 4],\n","       ...,\n","       [8, 0, 6, ..., 1, 9, 7],\n","       [2, 0, 1, ..., 5, 7, 9],\n","       [6, 5, 7, ..., 0, 2, 8]], dtype=int32))\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"lkKaMSEJnJUX","colab":{"base_uri":"https://localhost:8080/"},"outputId":"b1858b60-7b69-4b5d-f19d-72332c0b5776"},"source":["import tensorflow as tf\n","y_true = np.array(test_labels)\n","y_true = tf.identity(y_true)\n","y_pred = np.array(final_predictions)\n","y_pred = tf.identity(y_pred)\n","print(y_pred.shape,y_true.shape)\n","k = 8\n","recall, update_recall = tf.compat.v1.metrics.recall_at_top_k(y_true, y_pred, 5)\n","precision, update_precision = tf.compat.v1.metrics.precision_at_top_k(y_true, y_pred, 5)\n","\n","tmp_rank = tf.nn.top_k(y_pred, 5)\n","stream_vars = [i for i in tf.local_variables()]\n","\n","with tf.Session() as sess:\n","    sess.run(tf.local_variables_initializer())\n","    print(\"precision\",sess.run(update_precision))\n","    # print(\"precision\",sess.run(precision))\n","\n","    print(\"update_recall: \",sess.run(update_recall ))\n","    print(\"recall\",sess.run(recall))\n","\n","    print(\"STREAM_VARS: \",(sess.run(stream_vars)))\n","    print(\"TMP_RANK: \",sess.run(y_pred))"],"execution_count":null,"outputs":[{"output_type":"stream","text":["(1400, 5) (1400,)\n","precision 0.134\n","update_recall:  0.67\n","recall 0.67\n","STREAM_VARS:  [0.0, 0.0, 0.0, 0.0, 938.0, 462.0, 938.0, 6062.0]\n","TMP_RANK:  [[349 341 340 350 346]\n"," [ 47  45  44  46 282]\n"," [297  84 268 287  83]\n"," ...\n"," [194 188 189 190 192]\n"," [137 136 138 125 124]\n"," [  7   9   6   8  26]]\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"oPqYvRNIrRg1","colab":{"base_uri":"https://localhost:8080/"},"outputId":"2c64a8ae-abd6-4bd2-8b09-658785fa263a"},"source":["import tensorflow as tf\n","y_true = np.array(test_labels)\n","y_true = tf.identity(y_true)\n","y_pred = np.array(final_predictions)\n","y_pred = tf.identity(y_pred)\n","print(y_pred.shape,y_true.shape)\n","k = 8\n","recall, update_recall = tf.compat.v1.metrics.recall_at_top_k(y_true, y_pred, 1)\n","precision, update_precision = tf.compat.v1.metrics.precision_at_top_k(y_true, y_pred, 1)\n","\n","tmp_rank = tf.nn.top_k(y_pred, 1)\n","stream_vars = [i for i in tf.local_variables()]\n","\n","with tf.Session() as sess:\n","    sess.run(tf.local_variables_initializer())\n","    print(\"precision\",sess.run(update_precision))\n","    # print(\"precision\",sess.run(precision))\n","\n","    print(\"update_recall: \",sess.run(update_recall ))\n","    print(\"recall\",sess.run(recall))\n","\n","    print(\"STREAM_VARS: \",(sess.run(stream_vars)))\n","    print(\"TMP_RANK: \",sess.run(tmp_rank))"],"execution_count":null,"outputs":[{"output_type":"stream","text":["(4784, 1) (4784,)\n","precision 0.517767558528428\n","update_recall:  0.517767558528428\n","recall 0.517767558528428\n","STREAM_VARS:  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 2477.0, 2307.0, 2477.0, 2307.0]\n","TMP_RANK:  TopKV2(values=array([[ 66],\n","       [237],\n","       [162],\n","       ...,\n","       [ 49],\n","       [  8],\n","       [152]]), indices=array([[0],\n","       [0],\n","       [0],\n","       ...,\n","       [0],\n","       [0],\n","       [0]], dtype=int32))\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"TdD0JiYgEX4k"},"source":["!cp -r /content/model_euclidean_USE_cos_QC \"/content/drive/My Drive/research_lo_content_taxonomy_classification\""],"execution_count":null,"outputs":[]}]}