Soloupis_Greek_Classification_BERT.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Soloupis_Greek_Classification_BERT.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uQLp_6RGdrtw"
      },
      "source": [
        "This Colaboratory notebook shows a working workflow for **Text classification** with the [BERT](https://en.wikipedia.org/wiki/BERT_(language_model) model. It uses a [greek version ](https://huggingface.co/nlpaueb/bert-base-greek-uncased-v1) of a BERT pre-trained language model. Executing the cells one by one will give you a saved model folder that you can use later."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2h9ebJDQK_lC",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "752cadf0-4873-4001-86f9-7e164b8428a7"
      },
      "source": [
        "#Check if a GPU is present. If not go to Edit -> Notebook settings and change it to GPU for faster results\n",
        "\n",
        "!nvidia-smi"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sun May 23 04:27:59 2021       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 465.19.01    Driver Version: 460.32.03    CUDA Version: 11.2     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla T4            Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   68C    P0    30W /  70W |   1406MiB / 15109MiB |      0%      Default |\n",
            "|                               |                      |                  N/A |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4qdoPd6m4PjW"
      },
      "source": [
        "#Import neccessary libraries\n",
        "\n",
        "import os\n",
        "import shutil\n",
        "\n",
        "import tensorflow as tf\n",
        "import tensorflow_hub as hub\n",
        "import matplotlib.pyplot as plt\n",
        "import tensorflow.keras as K\n",
        "from keras.utils import to_categorical\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "tf.get_logger().setLevel('ERROR')"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "O0SXqw2g4cEi",
        "outputId": "a68c55f5-dc9f-4320-f5cc-d414b7c6b46c"
      },
      "source": [
        "print(tf.__version__)\n",
        "print(K.__version__)"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2.4.1\n",
            "2.4.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nayntnIB4cnV"
      },
      "source": [
        "#Install transformers and download the greek specific file for model and tokenizer from Hugging Face Hub\n",
        "\n",
        "!pip install transformers\n",
        "from transformers import AutoTokenizer, TFAutoModel\n",
        "from transformers import AdamWeightDecay\n",
        "\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"nlpaueb/bert-base-greek-uncased-v1\")\n",
        "model = TFAutoModel.from_pretrained(\"nlpaueb/bert-base-greek-uncased-v1\")\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "48NMmZlL4zUz",
        "outputId": "9f598100-42a4-439e-d464-05c72f952879"
      },
      "source": [
        "#View some info for the model and the tokenizer\n",
        "\n",
        "model.summary()\n",
        "print()\n",
        "model\n",
        "print()\n",
        "tokenizer"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"tf_bert_model_1\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "bert (TFBertMainLayer)       multiple                  112921344 \n",
            "=================================================================\n",
            "Total params: 112,921,344\n",
            "Trainable params: 112,921,344\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "PreTrainedTokenizerFast(name_or_path='nlpaueb/bert-base-greek-uncased-v1', vocab_size=35000, model_max_len=1000000000000000019884624838656, is_fast=True, padding_side='right', special_tokens={'unk_token': '[UNK]', 'sep_token': '[SEP]', 'pad_token': '[PAD]', 'cls_token': '[CLS]', 'mask_token': '[MASK]'})"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YANAif3_mOxZ"
      },
      "source": [
        "#GRGE example\n",
        "We are going to use a .csv file with greek texts from twitter that are divided in 3 categories (Negative, POsitive, Neutral)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wKvtu0TGmUgW",
        "outputId": "97cd7f33-1c23-47a3-a792-267a36209f8f"
      },
      "source": [
        "df = pd.read_csv(\"/content/GRGE_sentiment.csv\")\n",
        "print(df)"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "      Unnamed: 0                                               Text Polarity1\n",
            "0              0  Επιχείρηση Βολευτική έδρα!  Βασίλης Οικονόμου:...  Negative\n",
            "1              1  Κ ά β ο ς: Ψηφοδέλτιο ΣΥΡΙΖΑ Κορινθίας - Πού π...  Negative\n",
            "2              2  @Dora_Bakoyannis Αυτο-καρφώνεσαι όταν αναρωτιέ...  Positive\n",
            "3              3  Κόμμα «σκαντζόχοιρος»: «Σφαγή» πασοκογενών από...   Neutral\n",
            "4              4  Και γαμώ και την Νέα Δημοκρατία γιατί ειμαι πο...  Negative\n",
            "...          ...                                                ...       ...\n",
            "1635        1635  Κουτσούμπας: Δεν θα γίνουμε κερασάκι στην αντι...  Negative\n",
            "1636        1636  Θέλουμε μια Ελλάδα με πολιτικούς σαν την κ.Φωτ...  Negative\n",
            "1637        1637  Για διαχωρισμό κράτους-εκκλησίας, ομόφυλο γάμο...  Negative\n",
            "1638        1638  ΓΙΑΝΝΕΝΑ:Στην πρώτη γραμμή της προεκλογικής μά...   Neutral\n",
            "1639        1639  Σημαντικό βήμα για την αποφυγή της επερχόμενης...   Neutral\n",
            "\n",
            "[1640 rows x 3 columns]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OS3kqGiKmkGA",
        "outputId": "536492e3-ffa2-43fa-db1d-a78efc83541e"
      },
      "source": [
        "#View frst 10 entries\n",
        "\n",
        "text_list = df['Text'].to_list()\n",
        "print(text_list[:10])\n",
        "polarity = df[\"Polarity1\"].to_list()\n",
        "print(polarity[:10])"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['Επιχείρηση Βολευτική έδρα!  Βασίλης Οικονόμου:  Από το ΠΑΣΟΚ διαγράφηκε απο το ΓΑΠ επειδη ΔΕΝ ψήφισε το Μνημόνιο... http://t.co/gPgr1Caqxv', 'Κ ά β ο ς: Ψηφοδέλτιο ΣΥΡΙΖΑ Κορινθίας - Πού πας,ΣΥΡΙΖΑ, με δράμι μυαλό... http://t.co/8eIiKYS7CP', '@Dora_Bakoyannis Αυτο-καρφώνεσαι όταν αναρωτιέσαι ποιον ΠτΔ θα προτείνει ο ΣΥΡΙΖΑ  αποδεχόμενη την νίκη του αντιπάλου.', 'Κόμμα «σκαντζόχοιρος»: «Σφαγή» πασοκογενών από τις λίστες του ΣΥΡΙΖΑ http://t.co/rC7EY67qFY', 'Και γαμώ και την Νέα Δημοκρατία γιατί ειμαι πολιτικο ακκάουντ στην τελική όχι κλαψομούνικο!!!!', 'Ε. Καϊλή: «Ηθικό ζήτημα που θα κριθεί από τον κόσμο» η μετακίνηση της Γκερέκου στη ΝΔ http://t.co/Q4oYSfJjZZ', 'ΔΙΑΒΑΣΤΕ ΣΗΜΕΡΑ ΣΤΗΝ KONTRA NEWS : ΣΙΓΟΥΡΗ ΠΛΕΟΝ Η ΑΥΤΟΔΥΝΑΜΙΑ ΤΟΥ ΣΥΡΙΖΑ ΚΑΘΑΡΙΣΕ Ο ΣΑΜΑΡΑΣ ΓΙΑ ΤΟΝ ΑΛΕΞΗ ΔΥΟ... http://t.co/JZV7IHUTw1', 'Συμφωνεί ο ΣΥΡΙΖΑ στα Εθνικά θέματα με τον Τρεμόπουλο; Ο “οικολόγος” Μιχ. Τρεμόπουλος, εκείνος που είχε αναφέρ... http://t.co/nqFAZKAQO5', 'Τα μέτρα του ΣΥΡΙΖΑ ενάντια στη διαπλοκή http://t.co/XklqakuOyx via @youtube', 'Η επιστολή αποχώρησης από το ΠΑΣΟΚ για τον\\xa0Παπανδρέου http://t.co/lq9Fq6ylxq']\n",
            "['Negative', 'Negative', 'Positive', 'Neutral', 'Negative', 'Neutral', 'Neutral', 'Neutral', 'Neutral', 'Neutral']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Cpe5adItmvc1",
        "outputId": "97f587e2-a957-44ce-f724-b7c90c43d6d0"
      },
      "source": [
        "# Prepare input data for training\n",
        "\n",
        "def my_func(arg):\n",
        "  arg = tf.convert_to_tensor(arg, dtype=tf.string)\n",
        "  return arg\n",
        "\n",
        "train_x = my_func(np.array(text_list))\n",
        "print(train_x[:2])"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tf.Tensor(\n",
            "[b'\\xce\\x95\\xcf\\x80\\xce\\xb9\\xcf\\x87\\xce\\xb5\\xce\\xaf\\xcf\\x81\\xce\\xb7\\xcf\\x83\\xce\\xb7 \\xce\\x92\\xce\\xbf\\xce\\xbb\\xce\\xb5\\xcf\\x85\\xcf\\x84\\xce\\xb9\\xce\\xba\\xce\\xae \\xce\\xad\\xce\\xb4\\xcf\\x81\\xce\\xb1!  \\xce\\x92\\xce\\xb1\\xcf\\x83\\xce\\xaf\\xce\\xbb\\xce\\xb7\\xcf\\x82 \\xce\\x9f\\xce\\xb9\\xce\\xba\\xce\\xbf\\xce\\xbd\\xcf\\x8c\\xce\\xbc\\xce\\xbf\\xcf\\x85:  \\xce\\x91\\xcf\\x80\\xcf\\x8c \\xcf\\x84\\xce\\xbf \\xce\\xa0\\xce\\x91\\xce\\xa3\\xce\\x9f\\xce\\x9a \\xce\\xb4\\xce\\xb9\\xce\\xb1\\xce\\xb3\\xcf\\x81\\xce\\xac\\xcf\\x86\\xce\\xb7\\xce\\xba\\xce\\xb5 \\xce\\xb1\\xcf\\x80\\xce\\xbf \\xcf\\x84\\xce\\xbf \\xce\\x93\\xce\\x91\\xce\\xa0 \\xce\\xb5\\xcf\\x80\\xce\\xb5\\xce\\xb9\\xce\\xb4\\xce\\xb7 \\xce\\x94\\xce\\x95\\xce\\x9d \\xcf\\x88\\xce\\xae\\xcf\\x86\\xce\\xb9\\xcf\\x83\\xce\\xb5 \\xcf\\x84\\xce\\xbf \\xce\\x9c\\xce\\xbd\\xce\\xb7\\xce\\xbc\\xcf\\x8c\\xce\\xbd\\xce\\xb9\\xce\\xbf... http://t.co/gPgr1Caqxv'\n",
            " b'\\xce\\x9a \\xce\\xac \\xce\\xb2 \\xce\\xbf \\xcf\\x82: \\xce\\xa8\\xce\\xb7\\xcf\\x86\\xce\\xbf\\xce\\xb4\\xce\\xad\\xce\\xbb\\xcf\\x84\\xce\\xb9\\xce\\xbf \\xce\\xa3\\xce\\xa5\\xce\\xa1\\xce\\x99\\xce\\x96\\xce\\x91 \\xce\\x9a\\xce\\xbf\\xcf\\x81\\xce\\xb9\\xce\\xbd\\xce\\xb8\\xce\\xaf\\xce\\xb1\\xcf\\x82 - \\xce\\xa0\\xce\\xbf\\xcf\\x8d \\xcf\\x80\\xce\\xb1\\xcf\\x82,\\xce\\xa3\\xce\\xa5\\xce\\xa1\\xce\\x99\\xce\\x96\\xce\\x91, \\xce\\xbc\\xce\\xb5 \\xce\\xb4\\xcf\\x81\\xce\\xac\\xce\\xbc\\xce\\xb9 \\xce\\xbc\\xcf\\x85\\xce\\xb1\\xce\\xbb\\xcf\\x8c... http://t.co/8eIiKYS7CP'], shape=(2,), dtype=string)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "f2Ut2r5Rmwxg",
        "outputId": "896d381a-885c-4905-89f4-4a97670ba5fe"
      },
      "source": [
        "#Prepare labels for training\n",
        "\n",
        "encoded_labels = []\n",
        "for i in polarity:\n",
        "  if i == 'Negative':\n",
        "    encoded_labels.append(0)\n",
        "  elif i == 'Positive':\n",
        "    encoded_labels.append(1)\n",
        "  elif i == 'Neutral':\n",
        "    encoded_labels.append(2)\n",
        "    \n",
        "print(encoded_labels[:4])\n",
        "print(len(encoded_labels))\n",
        "\n",
        "transformed_encoded_labels = np.array(encoded_labels)\n",
        "print(transformed_encoded_labels[:4])"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[0, 0, 1, 2]\n",
            "1640\n",
            "[0 0 1 2]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4Q1UaNy8m5ba",
        "outputId": "81050eb4-33cf-4fb1-e5a4-dbe9cfc0f243"
      },
      "source": [
        "#Convert labels to categorical and then to tensor\n",
        "\n",
        "labels_split_categorical = to_categorical(encoded_labels)\n",
        "print(labels_split_categorical)\n",
        "\n",
        "# Prepare the labels data\n",
        "def my_func_float(arg):\n",
        "  arg = tf.convert_to_tensor(arg, dtype=tf.float32)\n",
        "  return arg\n",
        "\n",
        "train_labels = my_func_float(labels_split_categorical) \n",
        "print(train_labels)"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[1. 0. 0.]\n",
            " [1. 0. 0.]\n",
            " [0. 1. 0.]\n",
            " ...\n",
            " [1. 0. 0.]\n",
            " [0. 0. 1.]\n",
            " [0. 0. 1.]]\n",
            "tf.Tensor(\n",
            "[[1. 0. 0.]\n",
            " [1. 0. 0.]\n",
            " [0. 1. 0.]\n",
            " ...\n",
            " [1. 0. 0.]\n",
            " [0. 0. 1.]\n",
            " [0. 0. 1.]], shape=(1640, 3), dtype=float32)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pEBXWXNem8Kp"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fWfdbPGI7cGR"
      },
      "source": [
        "#Models"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RZuf10rW7ws1",
        "outputId": "3caeec37-393e-45d9-d27b-d16e7f5044f2"
      },
      "source": [
        "tfhub_handle_encoder = model\n",
        "tfhub_handle_preprocess = tokenizer\n",
        "\n",
        "print(f'BERT model selected: {tfhub_handle_encoder}')\n",
        "print(f'Preprocess model: {tfhub_handle_preprocess}')"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "BERT model selected: <transformers.models.bert.modeling_tf_bert.TFBertModel object at 0x7f23ef3940d0>\n",
            "Preprocess model: PreTrainedTokenizerFast(name_or_path='nlpaueb/bert-base-greek-uncased-v1', vocab_size=35000, model_max_len=1000000000000000019884624838656, is_fast=True, padding_side='right', special_tokens={'unk_token': '[UNK]', 'sep_token': '[SEP]', 'pad_token': '[PAD]', 'cls_token': '[CLS]', 'mask_token': '[MASK]'})\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7yaYek1U8FVu"
      },
      "source": [
        "bert_preprocess_model = hub.KerasLayer(tfhub_handle_preprocess)\n",
        "bert_model = hub.KerasLayer(tfhub_handle_encoder)"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wvoECpV-M4lK"
      },
      "source": [
        "#Test an input to the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WRwZfzqT99mv",
        "outputId": "a48aa8d6-5345-4131-943f-9320aa1dfe81"
      },
      "source": [
        "text_test = ['αυτή ήταν μια φανταστική ταινία!']\n",
        "text_preprocessed = bert_preprocess_model(text_test)\n",
        "#text_preprocessed = tokenizer.encode(text_test)\n",
        "print(text_preprocessed)\n",
        "print(f'Keys       : {list(text_preprocessed.keys())}')\n",
        "print(f'input_ids      : {text_preprocessed[\"input_ids\"][0]}')\n",
        "print(f'attention_mask   : {text_preprocessed[\"attention_mask\"][0][:12]}')\n",
        "print(f'token_type_ids   : {text_preprocessed[\"token_type_ids\"][0][:12]}')\n"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'input_ids': [[101, 393, 384, 369, 12859, 1146, 108, 102]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1]]}\n",
            "Keys       : ['input_ids', 'token_type_ids', 'attention_mask']\n",
            "input_ids      : [101, 393, 384, 369, 12859, 1146, 108, 102]\n",
            "attention_mask   : [1, 1, 1, 1, 1, 1, 1, 1]\n",
            "token_type_ids   : [0, 0, 0, 0, 0, 0, 0, 0]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3C1ouBk9-mjt",
        "outputId": "01d2b607-e353-4612-d77a-b505ca8afc8b"
      },
      "source": [
        "# Prepare input to be used from the model\n",
        "# Generate a dictionary of tensor inputs\n",
        "\n",
        "def my_func(arg):\n",
        "  arg = tf.convert_to_tensor(arg, dtype=tf.int32)\n",
        "  return arg\n",
        "\n",
        "transformed_text = {'input_ids': my_func(text_preprocessed[\"input_ids\"]), 'token_type_ids': my_func(text_preprocessed[\"token_type_ids\"]), 'attention_mask': my_func(text_preprocessed[\"attention_mask\"])}\n",
        "print(transformed_text)"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'input_ids': <tf.Tensor: shape=(1, 8), dtype=int32, numpy=\n",
            "array([[  101,   393,   384,   369, 12859,  1146,   108,   102]],\n",
            "      dtype=int32)>, 'token_type_ids': <tf.Tensor: shape=(1, 8), dtype=int32, numpy=array([[0, 0, 0, 0, 0, 0, 0, 0]], dtype=int32)>, 'attention_mask': <tf.Tensor: shape=(1, 8), dtype=int32, numpy=array([[1, 1, 1, 1, 1, 1, 1, 1]], dtype=int32)>}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "muTTlNlr_Bxl",
        "outputId": "22aaafca-a6f7-4c1a-fac3-2120fcf4fa6a"
      },
      "source": [
        "# Get the result of the model after passing the dictionary\n",
        "\n",
        "bert_results = bert_model(transformed_text)\n",
        "#print(bert_results)\n",
        "print(f'Loaded BERT: {tfhub_handle_encoder}')\n",
        "print(f'Keys       : {list(bert_results.keys())}')\n",
        "print(f'Pooled Output Shape:{bert_results[\"pooler_output\"].shape}')\n",
        "print(f'Pooled Output Values:{bert_results[\"pooler_output\"][0, :12]}')\n",
        "print(f'Sequence Output Shape:{bert_results[\"last_hidden_state\"].shape}')\n",
        "print(f'Sequence Output Values:{bert_results[\"last_hidden_state\"][0, :12]}')"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Loaded BERT: <transformers.models.bert.modeling_tf_bert.TFBertModel object at 0x7f23ef3940d0>\n",
            "Keys       : ['last_hidden_state', 'pooler_output']\n",
            "Pooled Output Shape:(1, 768)\n",
            "Pooled Output Values:[-0.7756876  -0.48410642  0.10253016  0.621603   -0.46155113  0.10883062\n",
            "  0.961941    0.41196406  0.6992992  -0.42222884 -0.2164939  -0.2050891 ]\n",
            "Sequence Output Shape:(1, 8, 768)\n",
            "Sequence Output Values:[[-0.95883304 -0.26517636  0.12192754 ... -0.74791586  0.35750234\n",
            "   1.0346323 ]\n",
            " [-1.3804331   0.2030597  -0.02667625 ...  1.0282251   0.3847218\n",
            "  -2.057902  ]\n",
            " [-0.9816262   0.25108054  1.1753194  ... -0.28084525  0.37680596\n",
            "  -0.6475431 ]\n",
            " ...\n",
            " [-1.4601182   0.45342112 -0.0151363  ...  0.8060957  -0.05657317\n",
            "  -1.1681705 ]\n",
            " [-1.0056059  -0.18209678 -0.11410159 ...  0.5029514  -0.15278082\n",
            "   0.19413413]\n",
            " [-0.07957786  0.1275501   1.0447783  ... -0.039065    0.3833198\n",
            "   0.3138709 ]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gT873yG1_ZqT"
      },
      "source": [
        "#Encode text dataset\n",
        "We have to preprocess all the inputs so they are ready to be fed to the network."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6b0Tbc7a_Cja"
      },
      "source": [
        "#Create 3 inputs for the model in numpy array format\n",
        "\n",
        "def bert_encode(texts, tokenizer, max_len=128):\n",
        "    all_tokens = []\n",
        "    all_masks = []\n",
        "    all_segments = []\n",
        "    \n",
        "    for text in texts:\n",
        "        text_preprocessed = tokenizer(text)\n",
        "        tokenized_text = text_preprocessed[\"input_ids\"]\n",
        "        #print(tokenized_text)\n",
        "        pad_len = max_len - len(tokenized_text)\n",
        "        #print(pad_len)\n",
        "\n",
        "        tokens = text_preprocessed[\"input_ids\"] + [0] * pad_len\n",
        "        pad_masks = [1] * len(tokenized_text) + [0] * pad_len\n",
        "        segment_ids = [0] * max_len\n",
        "\n",
        "        all_tokens.append(tokens)\n",
        "        all_masks.append(pad_masks)\n",
        "        all_segments.append(segment_ids)\n",
        "    \n",
        "    return np.array(all_tokens), np.array(all_masks), np.array(all_segments)"
      ],
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "P-luFO7TAXJQ",
        "outputId": "76b1e7c1-9a0f-4eb5-8b32-b15f473caeaa"
      },
      "source": [
        "# Encode all text\n",
        "\n",
        "input_x = bert_encode(text_list, bert_preprocess_model)\n",
        "print(input_x[:4])"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(array([[  101,  1314, 16357, ...,     0,     0,     0],\n",
            "       [  101,   242,   233, ...,     0,     0,     0],\n",
            "       [  101,   149,  2580, ...,     0,     0,     0],\n",
            "       ...,\n",
            "       [  101,   353, 17043, ...,     0,     0,     0],\n",
            "       [  101, 14877,   143, ...,     0,     0,     0],\n",
            "       [  101,   735,  1121, ...,     0,     0,     0]]), array([[1, 1, 1, ..., 0, 0, 0],\n",
            "       [1, 1, 1, ..., 0, 0, 0],\n",
            "       [1, 1, 1, ..., 0, 0, 0],\n",
            "       ...,\n",
            "       [1, 1, 1, ..., 0, 0, 0],\n",
            "       [1, 1, 1, ..., 0, 0, 0],\n",
            "       [1, 1, 1, ..., 0, 0, 0]]), array([[0, 0, 0, ..., 0, 0, 0],\n",
            "       [0, 0, 0, ..., 0, 0, 0],\n",
            "       [0, 0, 0, ..., 0, 0, 0],\n",
            "       ...,\n",
            "       [0, 0, 0, ..., 0, 0, 0],\n",
            "       [0, 0, 0, ..., 0, 0, 0],\n",
            "       [0, 0, 0, ..., 0, 0, 0]]))\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "B9ZFxeWTAsbk"
      },
      "source": [
        "#Build the model\n",
        "Below we build the model and we use two Dense layers and one Dropout layer."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VZ3MDDsiAbpj"
      },
      "source": [
        "def build_model(bert_layer, max_len=128):\n",
        "    input_word_ids = tf.keras.Input(shape=(max_len,), dtype=tf.int32, name=\"input_ids\")\n",
        "    input_mask = tf.keras.Input(shape=(max_len,), dtype=tf.int32, name=\"attention_mask\")\n",
        "    segment_ids = tf.keras.Input(shape=(max_len,), dtype=tf.int32, name=\"token_type_ids\")\n",
        "\n",
        "    bert_results = bert_layer([input_word_ids, input_mask, segment_ids])\n",
        "    #output = bert_results[\"last_hidden_state\"][:, 0, :]\n",
        "    output = bert_results[\"pooler_output\"][:,:]\n",
        "    net = tf.keras.layers.Dense(32, activation='relu')(output)\n",
        "    net = tf.keras.layers.Dropout(0.2)(net)\n",
        "    out = tf.keras.layers.Dense(3, activation='softmax')(net)\n",
        "    \n",
        "    model = tf.keras.models.Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)\n",
        "    \n",
        "    return model"
      ],
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OtZlhmS2AuaO",
        "outputId": "bbe90afe-f482-417a-844f-af530dc155b0"
      },
      "source": [
        "model = build_model(bert_model)\n",
        "model.summary()"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_ids (InputLayer)          [(None, 128)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "attention_mask (InputLayer)     [(None, 128)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "token_type_ids (InputLayer)     [(None, 128)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "keras_layer_1 (KerasLayer)      TFBaseModelOutputWit 112921344   input_ids[0][0]                  \n",
            "                                                                 attention_mask[0][0]             \n",
            "                                                                 token_type_ids[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "tf.__operators__.getitem (Slici (None, 768)          0           keras_layer_1[0][1]              \n",
            "__________________________________________________________________________________________________\n",
            "dense (Dense)                   (None, 32)           24608       tf.__operators__.getitem[0][0]   \n",
            "__________________________________________________________________________________________________\n",
            "dropout_74 (Dropout)            (None, 32)           0           dense[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "dense_1 (Dense)                 (None, 3)            99          dropout_74[0][0]                 \n",
            "==================================================================================================\n",
            "Total params: 112,946,051\n",
            "Trainable params: 24,707\n",
            "Non-trainable params: 112,921,344\n",
            "__________________________________________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 564
        },
        "id": "9oBB4mfaOYx7",
        "outputId": "4c647e08-d63d-433d-fa44-02f712451a3d"
      },
      "source": [
        "tf.keras.utils.plot_model(model)"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAs0AAAIjCAYAAAD4ASZzAAAABmJLR0QA/wD/AP+gvaeTAAAgAElEQVR4nOzdeXwV9b3/8ffJerInLEmAECABQRYXiC0i3Lpxi/WKYCDsFFoMYL2iYhtcLpdrRYhgwYqIEbSCFsLiheIKuCBYpBJRkCVGtgARwhISJIFs398fXs7PQMichCST5fV8PPIHc+bMfOY7M9+8mfM93ziMMUYAAAAArsjD7gIAAACAuo7QDAAAAFggNAMAAAAWCM0AAACABa9LF2zZskV/+ctf7KgFsNWjjz6qm2++uUa2zX0F1A/0AwAk6eabb9ajjz5aZtllT5oPHz6slStX1lpRQF2wcuVKHT58uMa2z30F1H30AwAk6YsvvtCWLVsuW37Zk+aLVqxYUaMFAXWJw+Golf1wXwF1F/0AAEkaPHhwucsZ0wwAAABYIDQDAAAAFgjNAAAAgAVCMwAAAGCB0AwAAABYIDQDAAAAFgjNAAAAgAVCMwAAAGCB0AwAAABYIDQDAAAAFgjNAAAAgAVCMwAAAGCB0AwAAABYIDQDAAAAFqolNL/33nsKCQnR2rVrq2NztnHnOMaNG6egoCA5HA59/fXX1bbvL774Qtdee608PDzkcDgUERGhZ555ptq2Xx1WrVqlmJgYORwOORwORUZGauTIkXaXBRs1lHu/qmbMmKGQkJBq6w/oB1BdqvvabKzIBRVrbP2BV3VsxBhTHZuxnTvHsXDhQt15550aNmxYte67Z8+e2rNnj/r166cPP/xQ6enpCg0NrdZ9XK34+HjFx8erffv2OnnypI4dO2Z3SbBZQ7n3q+rxxx9Xu3btqq0/oB9Adanua7OxIhdUrLH1B9XypPnuu+9Wbm6u7rnnnurY3FUpKChQr169qvTeunQcdcHVtCVqXnnnpybPWXnb5p5p+OgH7NUQ278+HRN9XFn16dzVhAY3pnnRokXKzs6u0X04HI4a3X5dURttiaor7/zU5DnjemicOO/2aojt3xCPiVzQOFx1aN68ebOio6PlcDg0b948SdL8+fMVEBAgf39/rVmzRnfddZeCg4MVFRWlpUuXut7717/+VU6nU+Hh4ZowYYJatGghp9OpXr16aevWra71HnroIfn4+CgyMtK17A9/+IMCAgLkcDh08uRJSdLDDz+syZMna9++fXI4HGrfvv1VHYf000czs2bNUseOHeXr66uQkBD98Y9/vOz9Gzdu1C9+8Qv5+/srODhY3bp1U15eniTpgw8+UHBwsKZPn+52PRfVx7b8uU2bNqlz584KCQmR0+lUt27d9OGHH0r6aRzYxXFQsbGx2r59uyRp7Nix8vf3V0hIiP7xj39IkkpKSjR16lRFR0fLz89P1113nVJTUyVJzz33nPz9/RUUFKTs7GxNnjxZrVq1Unp6epVqrisqarvyzs+VzllFbefu9VXetiu6Z/7yl7/o2muvla+vr8LCwjRgwADt3bvXtY67+3XX3LlzFRAQIA8PD/Xo0UMRERHy9vZWQECAunfvrj59+qh169ZyOp0KDQ3Vn/70J7fbWqr4/r7U8ePH1bZtW3l5ealfv36u5fQD9AOVdaX2d+ceK8+Vrs3q6COu5pjcvQbcvb6tjsld5AL6g8uYS6SmpppyFlfo8OHDRpJ58cUXXcuefPJJI8l89NFHJjc312RnZ5s+ffqYgIAAU1hY6Fpv/PjxJiAgwOzevducP3/e7Nq1y9x0000mKCjIZGZmutYbMWKEiYiIKLPfWbNmGUnmxIkTrmXx8fEmNja2UvVbHYfD4TDPP/+8ycnJMfn5+eall14yksz27duNMcb8+OOPJjg42CQnJ5uCggJz7Ngxc99997nqeuedd0xQUJB5+umnLWv49a9/bSSZnJycMjXUpbaMjY01ISEhlsdijDErVqww06ZNM6dPnzanTp0yPXv2NE2bNi2zD09PT3P06NEy7xs+fLj5xz/+4fr3Y489Znx9fc3KlStNTk6OeeKJJ4yHh4f58ssvy7TRpEmTzIsvvmjuu+8+s2fPHrdqNMYYSSY1NdXt9SurKveVO2136fkpb5m7bWd1fZW37fLumalTpxofHx+zZMkSc+bMGbNjxw7TvXt306xZM3Ps2DHXeu7u113//d//bSSZrVu3mnPnzpmTJ0+afv36GUnm3XffNSdOnDDnzp0zDz30kJFkvv76a7fa2ur+Xrp0aZn+oLCw0MTHx5s1a9aUqY9+gH6gKv1Aee3v7j3m7rVZXX3E1RyTu9eAu9e31TG5i1zQOPuDQYMGmUGDBl22vMZDc0FBgWvZxYvq+++/dy0bP378ZQ395ZdfGknmf/7nf1zL7AjN+fn5xt/f3/Tt27fMepd2RN9++62RZN55550q7ffnKro56kpbVubmuNSzzz5rJJns7GxjjDEbNmwwkswzzzzjWic3N9d06NDBFBcXG2OMKSgoMP7+/mbo0KGudfLz842vr6954IEHjDHlt1Fl1MVflpe6tO3cCc1Vbbvyri93QnN+fr4JDAwssz9jjPnXv/5lJJX5BeHuft11MTSfPXvWteyNN94wkszOnTsvq2XZsmVX3NbP29rq/v55f1BUVGSGDRtm3n///UrX/3P0A/QDF13a/pW5x9y5Nquzj6jqMRnj3jVgjHvXtzvH5C5yQePsD64Ummt1TLOPj48kqaioqML14uLi5O/vb/lRU037/vvvlZ+frzvuuKPC9WJiYhQeHq6RI0dq2rRpOnjwYI3XVt/a8iJvb29JP32sIkm33367rrnmGr322muubykvW7ZMQ4cOlaenpyQpPT1d+fn56tq1q2s7fn5+ioyMrDPHVRsubTt3VLXt3L2+LrVr1y79+OOPiouLK7P8pptuko+Pz2UfoVbXfq22V1xc7Fp2sR0r2sfP29rd+7ukpETDhw9XeHh4mY++axL9QOPrB6pyj1V0bdZ2H3El7lwDV3Lp9V2T1wq5oPrVp/6gzn4R0NfXVydOnLC1hiNHjkiSmjdvXuF6fn5++vjjj9W7d29Nnz5dMTExGjp0qAoKCmqjTEt2tuW7776rW2+9Vc2bN5evr+9lY0kdDocmTJig/fv366OPPpIkLV68WL///e9d65w7d06S9NRTT7nGOjkcDh06dEj5+fm1dzC1zKrt3FHbbXfmzBlJUmBg4GWvhYaG6uzZs9W+z+pQUVu7e38/+OCDysjI0IIFC7R79+7aPgRL9AMNQ1XusYquzbrSru5cAxX5+fVdk8dELrh69bk/qJOhuaioSGfOnFFUVJStdTidTknShQsXLNft0qWL1q5dq6ysLCUlJSk1NVWzZ8+u6RIt1XZbfvbZZ5ozZ44kKTMzUwMHDlRkZKS2bt2q3NxcJScnX/aeMWPGyOl0auHChUpPT1dwcLDatGnjev1i5zRnzhyZn4YUuX62bNlSK8dV29xtOyu13XYX5xAt7xd3Xbiny+NOW7tzfyckJGj9+vUKDQ3V6NGjyzzdthv9QMNRlXusomuzLrWr1TVwJZde3zV5TOSCymtI/UGdDM2ffvqpjDHq2bOna5mXl1e1fQzkrq5du8rDw0MbN26scL2srCzX/96bN2+uGTNmqHv37nXiaVNtt2VaWpoCAgIkSTt37lRRUZEeeOABxcTEyOl0ljstT1hYmIYMGaLVq1dr9uzZuv/++8u8fnHWg8b0V63cbTsrtd12Xbt2VWBgoLZt21Zm+datW1VYWKgePXrUSh2VYdXW7t7ft912m5o1a6aUlBSlpaXVqb/cRT/QcFTlHqvo2qxL7Wp1DVzJpdd3TR4TuaDyGlJ/UCdCc2lpqXJyclRcXKwdO3bo4YcfVnR0tMaMGeNap3379jp9+rRWr16toqIinThxQocOHbpsW02aNFFWVpYOHjyos2fPXtVF0Lx5c8XHx2vlypVatGiR8vLytGPHDqWkpJRZLysrSxMmTNDevXtVWFio7du369ChQ64L8v3336/y1DKVZVdbFhUV6fjx4/r0009dN0d0dLQkacOGDTp//rwyMjKuOKZ14sSJunDhgt55553LJpF3Op0aO3asli5dqvnz5ysvL08lJSU6cuSIfvjhh8o2Ub3gTtuVd34uXebp6VltbefO9eB0OjV58mS9/fbbevPNN5WXl6edO3dq4sSJatGihcaPH1/1RqkhVm1tdX9fqn///hozZoymT5+utLQ013L6AfqBqijvnq7qPVbetWlHu1Z0TVV0DVxkdX3X5DGRCxp5f3DpNwMr++3eF1980URGRhpJxt/f3/Tv39+89NJLxt/f30gyHTp0MPv27TMpKSkmODjYSDJt2rQx3333nTHmp292ent7m1atWhkvLy8THBxsBgwYYPbt21dmP6dOnTK33XabcTqdpl27duY///M/zR//+EcjybRv3941dcpXX31l2rRpY/z8/Ezv3r3LTL9T2eMwxpizZ8+acePGmaZNm5rAwEDTu3dvM3XqVCPJREVFmW+++cYcPHjQ9OrVy4SFhRlPT0/TsmVL8+STT7q+5fnee++ZoKCgMt8EvdQXX3xhunTpYjw8PIwkExkZaaZPn16n2vLll182sbGxRlKFP2+//bZrX0lJSaZJkyYmNDTUDB482MybN89IMrGxsWWmuzHGmBtvvNE8/vjj5bbPhQsXTFJSkomOjjZeXl6mefPmJj4+3uzatcskJycbPz8/I8m0bt3aLFmyxK1z/nOqg9+at2q78q718pZV1HaVub4u3fZTTz1V7j1TWlpqZs2aZTp06GC8vb1NWFiYGThwoElPT3cdW2X26465c+e6tte2bVuzadMmM3PmTBMSEmIkmYiICPPWW2+ZZcuWmYiICCPJhIWFmaVLl1q29aZNm654f69atcqEhYW59pudnW3y8vJM69atjSQTGBhoFi9ebIyhH6AfqFo/UN497c49Vplrs7r6iKs5pp+r6Bpw9/qu6JjcRS5ovP1BjU45dzXGjx9vmjRpUmv7a8jqe1v+5je/Mfv377dl33XxlyVQFfQDVUc/UDdUdA3U9+u7ttX39rKrP6gTU85dSWWm0ELF6lNb/vxjnR07dsjpdKpdu3Y2VgQ0DPQDqE8qew3Up+u7LqhP7VXX+4M6EZpryt69e8tMRXKln6FDh9pdaqOUlJSkjIwMfffddxo7dqz+/Oc/210S6hDu38aBfqD+u9p7tTavAfqVuq2u9wdedu78iSee0Ouvv67CwkK1a9dOs2bN0qBBg6pt+506dXJNjN3Q1XRb1gR/f3916tRJrVq10ksvvaTOnTvbXRLqkMZ0/1YX+gHY4WrvVXevgeq4vhtTv0J/UP0c5pKrZ/ny5RoyZEijuagA6afJ1FNTU5WQkFAj2+e+Auo++gEAkjR48GBJ0ooVK8osb9DDMwAAAIDqQGgGAAAALBCaAQAAAAuEZgAAAMACoRkAAACwQGgGAAAALBCaAQAAAAuEZgAAAMACoRkAAACwQGgGAAAALBCaAQAAAAuEZgAAAMACoRkAAACw4HWlFwYPHlybdaCRyc7Olq+vr0JCQuwupVZxXwFoSP3AmTNnVFBQoBYtWthdClBtvvjiC/Xs2fOy5ZeF5tatW2vQoEG1UhQar4yMDP3www9q2rSpYmNjFRUVJQ8P+z74GDRokFq3bl1j2+e+ani2bdsmSYqLi7O5ElQX+gH3XLhwQZmZmTp48KByc3MVHh5OaEaD0rNnT918882XLXcYY4wN9QBKS0vTCy+8oGXLlikgIECjR4/WpEmTFBMTY3dpgKWEhARJ0vLly22uBKh5JSUl+uSTT5SSkqI1a9bI6XTq3nvv1ejRo3XHHXfI4XDYXSJQ4xjTDNv06NFDixcvVmZmpqZMmaI1a9aoQ4cO6tu3r1asWKHi4mK7SwSARm337t2aMmWKWrVqpV//+tfKysrSiy++qKNHj2rx4sW68847CcxoNAjNsF1kZKSSkpK0f/9+ffjhh3I6nRoyZIjatm2radOmKTs72+4SAaDRyMnJUUpKinr37q0uXbro73//u8aMGaOMjAxt3rxZiYmJCgwMtLtMoNYRmlFneHh46M4779TatWuVnp6ukSNHat68eWrdurUSEhK0YcMGMZoIAKpfSUmJNmzYoNGjR6tVq1Z6+OGH1bJlS61fv16HDh3SzJkzGTqHRo/QjDqpQ4cOmjlzpo4ePao333xTWVlZ6tu3rzp37qzk5GTl5OTYXSIA1Ht79uzRlClTFBUVpb59+2r37t2aO3eusrOztXz5coZfAD/DFwFRb6SlpSklJUVvvfWWPDw8NGzYMD3wwAO6/vrr7S4NjRBfBER9debMGS1fvlyLFy/W559/rqioKI0YMUL333+/YmNj7S4PqLN40ox6o0ePHnrllVd09OhRzZ49W59//rluuOEGxcXFKSUlRQUFBXaXCAB1UmlpaZnhF5MmTXINv8jMzNTMmTMJzIAFQjPqnZCQECUmJurbb7/Vpk2bFBMTowcffFBt27bVlClTdODAAbtLBIA6Ye/evZo2bZpiYmJcwy/mzJnD8AugCgjNqNd69+6t5cuX69ChQ3r00Uf197//Xe3bt3dNW1dSUmJ3iQBQq3Jzc7V48WLX90AWLlyooUOHKiMjQ9u2bVNiYqKCgoLsLhOodwjNaBBatGihpKQkHThwQKtXr5YkDRkyRG3atNG0adN04sQJmysEgJrz8+EXLVu21Pjx4xUWFqY1a9a4Zr9o37693WUC9RpfBESD9d133+m1117Tq6++qnPnzql///5KTEzUnXfeaXdpaAD4IiDqgvT0dC1dulRvvPGGDh48qB49emjUqFEaNWqUmjRpYnd5QIPCk2Y0WNdcc41r2rqUlBR9//33ro8rX3jhBf344492lwgAlZaXl+cafnHttdfq1VdfVf/+/bVjxw5t27ZNkyZNIjADNYDQjAbP6XRq9OjR+uqrr7Rt2zb16dNHTzzxhFq1aqXx48dr586ddpcIABUqLS3V5s2bNX78eLVs2VKJiYllhl+88MIL6tatm91lAg0aoRmNys+nrZs1a5Y2bdqk6667TnFxcVq8eLGKiorsLhEAXDIzM5WcnKz27durT58+SktL0/Tp03X06FEtX75c99xzj7y8vOwuE2gUCM1olEJDQ13T1q1fv14xMTH6/e9/r9atW2vKlCk6dOiQ3SUCaKR+Pvyibdu2euGFF3TPPffom2++cQ2/aNq0qd1lAo0OoRmNmoeHh+68807XtHWPPPKI3nrrLdecpkxbB6A2/Hz4RatWrZSYmCin06nU1FRlZmbqhRde0HXXXWd3mUCjRmgG/k/Lli2VlJSkffv2admyZZJ+mrauU6dOSk5O1smTJ22uEEBDc/jwYSUnJ6tDhw7q06ePNm/erKeeekpHjhzR2rVrNXjwYIZfAHUEU84BFUhPT9frr7+ulJQU5efnM20dXJhyDlVVUFCgd955RykpKfroo48UERGhhIQEjR07VjfccIPd5QG4Ap40AxXo2LGjZs6cqczMTP31r39VRkaG+vbtqx49eiglJUXnzp2zu0QA9URaWprGjx+viIgIjRw58rLhFwRmoG4jNANuCAwMVGJiorZv365t27YpLi5ODz/8sOsvb3377bd2lwigDjpy5Ihr9ou4uDht3rxZTz75ZJnhF97e3naXCcANDM8AqujMmTN644039MILL+jAgQO65ZZbNGnSJA0YMIBfgo0AwzNwJefPn9fatWu1ePFivf/++woJCdGgQYM0YcIE3XjjjXaXB6CKeNIMVFFoaKgmTZqk77//XuvXr1fLli01fPhwRUdHa8qUKcrMzLS7RAC16OLwi/DwcA0bNkznz5/X0qVLdezYMb3yyisEZqCeIzQDV+nn09YdPHhQ48eP1+uvv6527dqpb9++Wrt2rfhAB2iYjh49quTkZF1zzTVlhl9kZWVp/fr1DL8AGhBCM1CNWrVqpWnTpunw4cOuaevuvfdedezYUcnJyTp16pTNFQK4WufPn9eKFSt0zz33qE2bNkpOTtZtt92mbdu2adeuXUpKSlJ4eLjdZQKoZoRmoAb4+Pho8ODBWr9+vXbv3q377rtPM2fOVKtWrZSQkKDPP//c7hIBVFJaWpomTZqkqKiocodf9OjRw+4SAdQgQjNQwzp16lRm2rr09HT17t1bcXFxrvmfAdRNWVlZrr/GFxcXpw0bNuiPf/yjjh496hp+4ePjY3eZAGoBs2cANkhLS1NKSoqWLFkiHx8fDRkyRA899JC6dOlid2lwE7NnNFwXLlzQP/7xDy1evFgffPCBAgMDlZCQoFGjRql37952lwfAJjxpBmzQo0cPvfLKKzpw4IAef/xxrVu3Tt26dVPfvn21YsUKFRcX210i0Oj8fPjF0KFDdf78eS1atEhZWVl65ZVXCMxAI0doBmwUERGhpKQk7du3T+vWrVNYWJiGDRvmmrbu8OHDdpcINGg//PCD66/xxcXFad26dfrDH/6gAwcOaP369Ro9erT8/PzsLhNAHcDwDKCO2bdvn1599VW99tprOn36tO666y5NmjRJd9xxhxwOh93l4f8wPKP+unDhgtatW6clS5bof//3fxl+AcAtPGkG6pjY2FjNnDlThw8f1tKlS3X+/Hn17dtXnTp1UnJysk6fPm13iUC9tGvXLk2ZMkVRUVEaMGCAcnJytGjRIh09epThFwAs8aQZqAf27NmjBQsW6LXXXlNxcbEGDx6sRx55hL8wZiOeNNcPx44dU2pqqv72t7/p66+/VseOHTV06FCNHTtWbdq0sbs8APUIoRmoR/Ly8rRs2TLNmzdPO3fuVI8ePZSYmKiRI0fK39/f7vIaFUJz3VVYWKgPP/xQS5Ys0erVq+Xv76/+/ftr9OjRDHMCUGUMzwDqkeDgYCUmJmrHjh3atm2bOnfurAcffFCtWrXS+PHjtWfPHrtLBGxz6fCLrKwszZs3T0ePHtXixYt15513EpgBVBlPmoF67tixY3rjjTf08ssv6/Dhw7r99tuVmJiogQMHysvLy+7yGoS//e1vmjt3rkpKSlzLTpw4IUlq3ry5a5mnp6cefvhhjRkzprZLbLROnz6tlStX6pVXXtFXX32la665RsOGDdOYMWPUtm1bu8sD0IAQmoEGorS0VB9//LFeeOEFvfvuu2rRooVGjRqlBx98UFFRUXaXV6+lp6erU6dObq27Z88et9dF1ZSUlOiTTz5RSkqKVq9eLT8/P917770MvwBQowjNQAP0/fffa+HChVq0aJHy8vJ07733KjExkUBxFa677jp9++23ulKX6XA41LVrV+3YsaOWK2s8du3apSVLluj111/XyZMndfPNN2v06NEaMWKEAgIC7C4PQAPHmGagAWrfvr1mzpypI0eO6M0331ROTo769u2ra6+9VsnJycrJyanU9pjmTho9erQ8PT2v+LqXl5d++9vf1mJFjUNOTo5SUlLUu3dvde3aVX//+981duxYZWRkaPPmzUpMTCQwA6gVPGkGGom0tDSlpKTorbfekoeHh4YNG6aJEyfqhhtusHzvmDFj5O3trZdffrnRjpPOyspSVFRUhU+aMzMzGQpTjrfeektFRUVuj/X++fCLNWvWyNPTU//xH//BpyUAbEVoBhqZ3Nxcpaam6sUXX9S3337rmrZu1KhR5f654NOnT6tFixYqLCxUv379tGrVqkY7vV3v3r21ZcsWlZaWllnu4eGhm2++WZs3b7apsrrJGKP/+q//0rPPPqvu3btr27ZtFa6/Z88evfHGG/rb3/6mEydOuIZfDB8+XIGBgbVUNQCUj+EZQCMTEhKixMRE7dy5U5s2bVJMTIwefPBBtWzZUpMmTdKBAwfKrP/aa6+5QuKGDRt0yy23KDs7247SbTdq1Khyn3I6HA6NHj3ahorqrnPnzik+Pl4zZsyQMUZpaWnlTol45swZ1/CLzp076+9//7vGjBmj7777zjX8gsAMoC7gSTMA17R18+fP15EjR1zT1g0YMEAdO3bUwYMHXcMSvL29FRkZqY8++kgdOnSwufLadfr0aUVERKi4uLjMck9PTx0/flxNmza1qbK6JSsrS3fffbd27dqloqIiST9dN48++qhmzpzpmull8eLFWrlypYwxuueeexh+AaBOIzQDcCkqKtKaNWv08ssv65NPPlG7du20f//+y9bz9vZWUFCQ1q1bpx49ethQqX1+85vfaP369a7g7Onpqb59++r999+3ubK64euvv9Zdd92lU6dOuQLzRc2aNdOYMWP01ltv6dixY+rTp4/Gjh2rQYMG8TQZQJ3H8AwALt7e3ho0aJA++ugj7d69W2FhYeV+8a+oqEi5ubnq3bu33nvvPRsqtc/IkSPLjGk2xmjUqFE2VlR3rFixQj179tTJkycvC8ySdPLkSX3wwQe6//77lZGRoY0bN2rMmDEEZgD1Ak+aAZTr8OHDatu27WVfevs5D4+f/t+9YMEC3X///bVVmq3OnTunZs2a6fz585IkX19fnTx5slEHP2OMnnvuOT3++OOuf5fHy8tLAwYM0IoVK2qzPACoFjxpBlCulJSUCuclln76K4SlpaVKTEzUtGnTaqcwmwUEBKh///7y9vZ2hcDGHJjPnz+vESNG6IknnpAx5oqBWZKKi4u1evVqnTp1qhYrBIDqQWgGcJmioiK9/PLL5X7EfiVPP/20Jk6cqJKSkhqsrG4YMWKEiouLVVJSouHDh9tdjm1++OEH3XLLLVqxYkWFn0hcKjU1tQarAoCa0Tj/SgFgsyNHjuif//yn3WVc0ZYtW8p9Gujp6SmHwyGHw+F6qngxJBtjtGDBAm3btk2TJk2Sj49PbZdda0pKSuR0OmWM0Y8//qjly5fbXVKtO3DggGbMmKHc3FzLdT08PFwzYpSUlGj27Nlq1qxZTZfoll69evEHaQC4hTHNgA2WL1+uIUOG2F0G0OilpqYqISHB7jIA1AM8aQZsxP9Z669PPvlEDodDt956q92loIqYDxpAZRCaAaAKfvWrX9ldAgCgFhGaAaAKLlMKIbsAACAASURBVE63BwBoHOj1AQAAAAuEZgAAAMACoRkAAACwQGgGAAAALBCaAQAAAAuEZgAAAMACoRkAAACwQGgGAAAALBCaAQAAAAuEZgAAAMACoRkAAACwQGgGAAAALBCagXpgxowZCgkJkcPh0Ndff213OVUye/ZshYeHy+FwaMGCBXaXU2WlpaWaM2eOevXqVeVtrFq1SjExMXI4HHI4HGrdurUWLVrken3jxo1q1aqVHA6HIiMjlZKSUh2lV0utkZGRGjlypG31AIBdvOwuAIC1xx9/XO3atdOwYcPsLqXKHnvsMQ0YMEAdOnSwu5Qqy8jI0NixY/X555/r+uuvr/J24uPjFR8fr/bt2+vkyZM6fPhwmdf/7d/+Tb/5zW/k4eGhBQsWyOFwXG3pVXZprceOHbOtFgCwE6EZANzwzTff6Omnn9bEiRN17tw5GWNqZD+lpaUaN26cnE6nXnrpJVsDMwDg/2N4BgC44frrr9eqVas0YsQI+fr61sg+SktL9bvf/U7+/v6aP38+gRkA6hBCM1BPHT9+XG3btpWXl5f69evnWl5SUqKpU6cqOjpafn5+uu6665SamipJeu655+Tv76+goCBlZ2dr8uTJatWqldLT07Vp0yZ17txZISEhcjqd6tatmz788EPXdjdu3Khf/OIX8vf3V3BwsLp166a8vLyrPo6K9jtu3DjXWNrY2Fht375dkjR27Fj5+/srJCRE//jHP67quKvbBx98oODgYE2fPr1S7ystLdWYMWMUEhKiefPmXXG9+nB+G9o5BQCJ0AzUW02aNFFcXJzefvttffDBB67lU6ZM0XPPPac5c+bohx9+0D333KPhw4dr27Zt+tOf/qRHH31UP/74o5599lm1a9dOPXv2lDFGx48f15AhQ3Tw4EFlZWUpMDBQI0aMkCSdO3dO/fv316BBg3T69GllZGTommuuUWFh4VUfR0X7XbhwoeLj4+Xp6alNmzbpxhtvlCS9/vrrGjhwoN58803179//qo67upWUlEj6KQRX5j2jR4/WsmXL9Ic//KHCdevD+W1o5xQAJEkGQK1LTU01lb39li5daiSZ7du3m6KiIjNs2DDz/vvvl1mnoKDA+Pv7m6FDh7qW5efnG19fX/PAAw8YY4x58sknjSRTUFBQ4f6effZZI8lkZ2ebb7/91kgy77zzTqVqvlRGRoaRZF5++WW39muMMRs2bDCSzDPPPONaJzc313To0MEUFxcbY6r3uN3xy1/+0lx//fVXvZ3Y2FgTFBRkhg0bZrp3724kmS5dupgff/yx3PXtPL+xsbEmJCSkCkdZd8+pJJOamlql9wJofHjSDNQzJSUlGj58uMLDw8sMy5Ck9PR05efnq2vXrq5lfn5+ioyM1N69eyu1H29vb9f+YmJiFB4erpEjR2ratGk6ePDgVR+HO/uVpNtvv13XXHONXnvtNddTxGXLlmno0KHy9PSUVL3HXdvy8/P1q1/9SmlpaRo4cKB27dqlcePGlbtufT2/je2cAmiYCM1APfPggw8qIyNDCxYs0O7du8u8du7cOUnSU0895Ro36nA4dOjQIeXn51e43XfffVe33nqrmjdvLl9fX/3pT39yvebn56ePP/5YvXv31vTp0xUTE6OhQ4eqoKDgqo+nov1KksPh0IQJE7R//3599NFHkqTFixfr97//fbUct90CAwM1fvx4ST8NUYiJidGyZcs0Z86cy9atL+e3sZ9TAA0ToRmoZxISErR+/XqFhoZq9OjRKi4udr3WvHlzSdKcOXNkjCnzs2XLlituMzMzUwMHDlRkZKS2bt2q3NxcJScnl1mnS5cuWrt2rbKyspSUlKTU1FTNnj37qo7Fnf1K0pgxY+R0OrVw4UKlp6crODhYbdq0uerjrmtCQkK0YsUKV9D87LPPyrxeV8/vZ5995gr5nFMADRWhGahnbrvtNjVr1kwpKSlKS0vTM88843qtdevWcjqdlf6rgTt37lRRUZEeeOABxcTEyOl0lpnuLCsry/VUu3nz5poxY4a6d+9+2ZPuyrLa70VhYWEaMmSIVq9erdmzZ+v+++8v83pVj7su6t69u+bMmaPi4mIlJCQoKyvL9VpdPb9paWkKCAhwa18XNaZzCqBhIDQD9VT//v01ZswYTZ8+XWlpaZIkp9OpsWPHaunSpZo/f77y8vJUUlKiI0eO6IcffrjitqKjoyVJGzZs0Pnz55WRkaGtW7e6Xs/KytKECRO0d+9eFRYWavv27Tp06JB69ux5Vcdgtd+fmzhxoi5cuKB33nlH99xzT5nXqnrcNeH999+v0pRzPzdx4kQNGzZMx48f1+DBg1VUVCSp7p3foqIiHT9+XJ9++qkrNDfEcwoAkpg9A7BDZWfPWLVqlQkLCzOSTNu2bU12drbJy8szrVu3NpJMYGCgWbx4sTHGmAsXLpikpCQTHR1tvLy8TPPmzU18fLzZtWuXSU5ONn5+fkaSad26tVmyZIlrH0lJSaZJkyYmNDTUDB482MybN89IMrGxsWbTpk2mV69eJiwszHh6epqWLVuaJ5980jXTgTuef/55ExERYSSZgIAAc99991nuNzMzs8w2brzxRvP444+Xu/2qHre7tmzZYm655RbTokULI8lIMpGRkaZXr15m48aNrvXee+89ExQUVGZmiEu9/fbbJjY21rWdqKgo88QTT5RZ5+zZs6Zjx45GkgkPDzeLFi26quOs6vm9tNYr/bz99ttu7asunVMxewaASnAYw6SWQG1bvny5hgwZwpyylXT33Xdr3rx5ateund2loJrYeU4dDodSU1OVkJBQ6/sGUP8wPANAnXVxWIIk7dixQ06nk8Bcz3FOAdRXhGYAVbZ3794y04Fd6Wfo0KFV2n5SUpIyMjL03XffaezYsfrzn/9cb2pH+WrynAJATfKyuwAA9VenTp1qdIiJv7+/OnXqpFatWumll15S586dq23bNV07yleT5xQAahJjmgEbMKYZsB9jmgFUBsMzAAAAAAuEZgAAAMACoRkAAACwQGgGAAAALBCaAQAAAAuEZgAAAMACoRkAAACwQGgGAAAALBCaAQAAAAuEZgAAAMACoRkAAACwQGgGAAAALBCaAQAAAAtedhcANGbLly+3uwQAAOAGQjNgoyFDhthdAgAAcIPDGGPsLgIA6puEhARJfFoAAI0FY5oBAAAAC4RmAAAAwAKhGQAAALBAaAYAAAAsEJoBAAAAC4RmAAAAwAKhGQAAALBAaAYAAAAsEJoBAAAAC4RmAAAAwAKhGQAAALBAaAYAAAAsEJoBAAAAC4RmAAAAwAKhGQAAALBAaAYAAAAsEJoBAAAAC4RmAAAAwAKhGQAAALBAaAYAAAAsEJoBAAAAC4RmAAAAwAKhGQAAALBAaAYAAAAsEJoBAAAAC4RmAAAAwAKhGQAAALBAaAYAAAAsEJoBAAAAC4RmAAAAwAKhGQAAALBAaAYAAAAsEJoBAAAAC152FwAAdd3GjRv1xRdflFm2d+9eSVJycnKZ5T179tSvfvWrWqsNAFA7HMYYY3cRAFCXrV+/Xv/+7/8ub29veXiU/wFdaWmpioqKtG7dOvXt27eWKwQA1DRCMwBYKCkpUUREhE6dOlXhemFhYcrOzpaXFx/iAUBDw5hmALDg6empESNGyMfH54rr+Pj4aNSoUQRmAGigCM0A4IZhw4apsLDwiq8XFhZq2LBhtVgRAKA2MTwDANzUpk0bZWZmlvtaVFSUMjMz5XA4arkqAEBt4EkzALhp5MiR8vb2vmy5j4+Pfvvb3xKYAaAB40kzALhpz5496ty5c7mv7dy5U127dq3ligAAtYXQDACV0LlzZ+3Zs6fMsk6dOl22DADQsDA8AwAqYfTo0WWGaHh7e+u3v/2tjRUBAGoDT5oBoBIyMzPVtm1bXew6HQ6H9u/fr7Zt29pbGACgRvGkGQAqITo6WnFxcfLw8JDD4dBNN91EYAaARoDQDACVNHr0aHl4eMjT01OjRo2yuxwAQC1geAYAVNKJEyfUokULSdLRo0cVERFhc0UAgJpGaEatGTx4sFauXGl3GQCAcgwaNEgrVqywuwygzvKyuwA0Lj179tQjjzxidxnAVdu4caMcDof+7d/+ze5SgKs2Z84cu0sA6jxCM2pVVFSUEhIS7C4DuGr9+vWTJAUHB9tcCXD1eMIMWCM0A0AVEJYBoHFh9gwAAADAAqEZAAAAsEBoBgAAACwQmgEAAAALhGYAAADAAqEZAAAAsEBoBgAAACwQmgEAAAALhGYAAADAAqEZAAAAsEBoBgAAACwQmgEAAAALhGYAAADAAqEZ9dKFCxc0adIkRUZGyt/fXx988IHdJaEee++99xQSEqK1a9faXYpbvvnmGw0dOlTt2rWTr6+vmjVrpuuvv17PPPOMa53Zs2crPDxcDodDCxYscC2vyrHWZvts2LBBgwYNUuvWreXr66vAwEB16dJFjzzyiA4dOlSlba5atUoxMTFyOBxyOByKjIzUyJEjq7ly99x0003y9PTUDTfcUOn3jhs3TkFBQXI4HPr6669roDoAFSE0o156/vnn9cEHH2jv3r2aO3eufvzxR7tLQj1mjLG7BLft3LlTvXr1UmRkpD755BPl5ubqn//8p/r166dPP/3Utd5jjz2mf/7zn5e9vyrHWlvtM2XKFPXt21fBwcFau3atcnNzlZWVpb/85S/atGmTrrvuOn388ceV3m58fLz279+v2NhYhYSE6NixY3rzzTdr4Aisffnll7rtttuq9N6FCxfq1VdfreaKALjLy+4CgIoUFBTojjvuuOyX/+rVqxUXF6fQ0FAlJibaVF3ddaV2Q/ltc/fddys3N9dyvbpg9uzZCg0N1dy5c13LrrnmGv35z39WfHy85fvLO9aaeE9lrVmzRsnJyUpMTNQrr7ziWu50OvXrX/9at9xyi3r06KGEhASlp6eradOmNVpPTXM4HHaXAKCSeNKMOm3RokXKzs6+bPmRI0fk7e1tQ0X1w5XaraExxmjFihVKSUlx+z3utk1dbcNTp04pNzdXp0+fLrPcx8en3gwvKe+8zZ49W5L01FNPlfuewMBAPfroozp16pQWLlxYK3XWpKr2X4RtwD6EZtRZDz/8sCZPnqx9+/bJ4XCoffv2Wr9+vdq3b68ffvhBb7zxhhwOhwIDA2utJmOM/vKXv+jaa6+Vr6+vwsLCNGDAAO3du9e1zl//+lc5nU6Fh4drwoQJatGihZxOp3r16qWtW7eW2V5JSYmmTp2q6Oho+fn56brrrlNqaqok6bnnnpO/v7+CgoKUnZ2tyZMnq1WrVkpPT9emTZvUuXNnhYSEyOl0qlu3bvrwww+v2G7u1l7RPjdu3Khf/OIX8vf3V3BwsLp166a8vLyabvIybfXss8+qY8eO8vPzU7NmzdSuXTs9++yzSkhIcKtNy2ubzZs3Kzo6Wg6HQ/PmzauwDSva9ty5cxUQECAPDw/16NFDERER8vb2VkBAgLp3764+ffqodevWcjqdCg0N1Z/+9Kcyx/fBBx8oODhY06dPr7AdbrrpJp07d0633367Pv/880q1YXnHetGSJUsUFxcnp9OpgIAAtW3bVn/+85/Lfc/8+fMVEBAgf39/rVmzRnfddZeCg4MVFRWlpUuXVvq85efn64svvlB0dLRat259xfpvvvlmSdL69eslVe5eq6yK7rGrPdeS9P3336tTp04KCAiQn5+f+vTpo82bN5dZxxijWbNmqWPHjvL19VVISIj++Mc/VqpWANXIALVk0KBBZtCgQZV6T3x8vImNjb1seUREhPntb39bTZW5b+rUqcbHx8csWbLEnDlzxuzYscN0797dNGvWzBw7dsy13vjx401AQIDZvXu3OX/+vNm1a5e56aabTFBQkMnMzHSt99hjjxlfX1+zcuVKk5OTY5544gnj4eFhvvzyS2OMMU8++aSRZCZNmmRefPFFc99995k9e/aYFStWmGnTppnTp0+bU6dOmZ49e5qmTZu6tlteu7lbe3n73LZtmwkODjbJycmmoKDAHDt2zNx3333mxIkTNdXUl5k+fbrx9PQ0a9asMfn5+SYtLc1ERESYW2+9tcx6Vm1aXtscPnzYSDIvvviia1l561lt+7//+7+NJLN161Zz7tw5c/LkSdOvXz8jybz77rvmxIkT5ty5c+ahhx4ykszXX3/t2vY777xjgoKCzNNPP11hO+Tn55u4uDgjyUgynTt3NsnJyebUqVOXrZuRkWEkmZdffrnCY50zZ46RZGbMmGFOnTplTp8+bV555RUzYsSIK77n4nXy0UcfmdzcXJOdnW369OljAgICTGFhYaXO2549e4wkExcXV+GxHz9+3Egy7dq1cy1z914zxpjY2FgTEhJS4T4usrrHruZc33HHHSYmJsYcOHDAFBUVmW+//db88pe/NE6n03z33Xdl2tjhcJjnn3/e5OTkmPz8fPPSSy8ZSWb79u1u1+qOqvTPQGNDaEatqe+hOT8/3wQGBpqhQ4eWWf6vf/3LSCoTdsaPH3/ZL+cvv/zSSDL/8z//Y4wxpqCgwPj7+5fZXn5+vvH19TUPPPCAMeb/B5OCgoIKa3v22WeNJJOdnW2MubzdKlN7efv89ttvjSTzzjvvVFhHTbrpppvML37xizLLEhMTjYeHh7lw4YIxxr02rWpodmfbF4PU2bNnXeu88cYbRpLZuXOna9nFdl+2bFmV2qKwsNC88MILplOnTq7wHB4ebj799NMy67kTmgsLC01oaKi57bbbyry3uLjYzJ0794rtU951cjHQff/9965l7py3i/fG7bffXuFxX7hwwUgyzZo1cy1z5167qDKh+VKX3mNXc67vuOMOc/3115fZ/o4dO4wk89hjjxljfrq2/P39Td++fcust3Tp0stCs1Wt7iA0A9YYngG4adeuXfrxxx8VFxdXZvlNN90kHx8fy4+D4+Li5O/v7xoOkZ6ervz8fHXt2tW1jp+fnyIjI8sMmXDHxfGRJSUlNVJ7TEyMwsPDNXLkSE2bNk0HDx6sVH3V4fz585fN4lBSUiJvb295enpKqt42vVRVt+3j4yNJKi4udi27eL6KioqqVIu3t7ceeugh7dmzR1988YUGDBig7OxsDR48WDk5OZXa1o4dO3TmzBn9+te/LrPc09NTkyZNqtS2Lh7rz4/LnfMWFBQkSTpz5kyF2784jjs4OLjC9S6916qD1T0mXd257tatm0JCQrRjxw5JPw3fyM/P1x133FEjtQKoPEIz4KaLv9DLG0MdGhqqs2fPWm7D19dXJ06ckCSdO3dO0k9ffLo4f6zD4dChQ4eUn59f4Xbeffdd3XrrrWrevLl8fX3LHTNZnbX7+fnp448/Vu/evTV9+nTFxMRo6NChKigoqPB91ek3v/mN0tLStGbNGhUUFGjbtm1avXq1/uM//sMVvq6mTa3U5Lavxi9/+Uv97//+ryZOnKgTJ07ok08+qdT7L45LDw0NrYny3Dpvbdq0kbe3t44fP17hto4dOyZJ6tChg+V+f36vVUVl77Hq4O3t7QrXR44ckSQ1b97c8n121Ao0RoRmwE0XQ0V5AfPMmTOKioqq8P1FRUVl1rv4y3DOnDkyPw2Vcv1s2bLlitvJzMzUwIEDFRkZqa1btyo3N1fJyck1WrskdenSRWvXrlVWVpaSkpKUmprqmvGgNkybNk233367xowZo+DgYN13331KSEgoM29tVdvUHTW57cqIj48v8yTzolGjRklSpQN8y5YtJUknT568+uLK4c55czqd6tOnj44ePaoDBw5ccVsXvyh36VPxS116r7njs88+05w5cyRV7R67WsXFxTp9+rSio6Ml/dQm0k9/yKkidtQKNFaEZsBNXbt2VWBgoLZt21Zm+datW1VYWKgePXpU+P5PP/1Uxhj17NlTklzfrq/sX/bauXOnioqK9MADDygmJkZOp9NyGqqrrT0rK0u7d++W9FN4nDFjhrp37+5aVht27dqlffv26cSJEyoqKlJmZqbmz5+vsLAw1zpVbVN31OS2K+PChQvltnt6erok6brrrqvU9tq2basmTZpo3bp11VLfpdw5b9JPf9hEkp5++ulyt5OXl6c5c+YoPDxcv/vd7yrc56X3mjvS0tIUEBAgqWr32NX65JNPVFpaqu7du0v66Z718PDQxo0bK3yfHbUCjRWhGXVakyZNlJWVpYMHD+rs2bMVjgucOnWqQkJCauyXv9Pp1OTJk/X222/rzTffVF5ennbu3KmJEyeqRYsWGj9+fJn1S0tLlZOTo+LiYu3YsUMPP/ywoqOjNWbMGNf2xo4dq6VLl2r+/PnKy8tTSUmJjhw5oh9++OGKdVx8ErVhwwadP39eGRkZl41JvrTdPD09K1X7pbKysjRhwgTt3btXhYWF2r59uw4dOlSpUFIZ5Z3LBx98UNHR0RX+9Ud32tTda6q8NqzK+XLX+++/79aUc5I0cOBALV++XGfOnFFubq7WrFmjKVOm6N577610aPb19dUTTzyhzz77TA899JCOHj2q0tJSnT17tlr+U+TOeZOkvn37asaMGXrjjTc0ZswYffPNNzp//rzy8vK0bt063XbbbcrJydHKlSsVEhJS5r1W91pFioqKdPz4cX366aeu0OzOPXa1CgsLlZubq+LiYn311Vd66KGH1KZNG1fNzZs3V3x8vFauXKlFixYpLy9PO3bsuGxO8tqoFcD/qf3vHqKxqsq3s7/66ivTpk0b4+fnZ3r37m22bt1qbrzxRiPJeHl5me7du5uVK1caY4z5r//6LxMUFGQ+/PDDmijfGGNMaWmpmTVrlunQoYPx9vY2YWFhZuDAgSY9Pb3MeuPHjzfe3t6mVatWxsvLywQHB5sBAwaYffv2lVnvwoULJikpyURHRxsvLy/TvHlzEx8fb3bt2mWSk5ONn5+fkWRat25tlixZ4npfUlKSadKkiQkNDTWDBw828+bNM5JMbGysyczMvKzdjh075lbtV9rnwYMHTa9evUxYWJjx9PQ0LVu2NE8++aQpLi6ukXYu71x+/PHHpmnTpq7ZIiQZb29vc+2115pVq1a51abGXH5NPfXUUyYyMtJIMv7+/qZ///7lrnfs2LEKtz137lzj7+9vJJm2bduaTZs2mZkzZ5qQkBAjyURERJi33nrLLFu2zERERBhJJiwszCxdutQYY8x7771ngoKCzDPPPFNh26xbt84MGTLExMbGGl9fX+Pj42M6duxopk2bZs6fP+9a7/nnn3ftJyAgwNx3333mxRdfLPdYjTFm3rx5plu3bsbpdBqn02luvPFG89JLL5X7npdeesl1rB06dDD79u0zKSkpJjg42Egybdq0cU2d5u55u2jLli1m+PDhJjo62vj4+JiAgADTtWtXM3nyZHPkyJHL1nfnXnv77bdNbGxsmRrK+3n77bdd76noHps8efJVnevXX3/d3HbbbSY8PNx4eXmZpk2bmmHDhplDhw6VObazZ8+acePGmaZNm5rAwEDTu3dvM3XqVCPJREVFmW+++cay1kun3bsSZs8ArDmMueRrzUANGTx4sCRpxYoVNldS8yZMmKAVK1bo1KlTdpfSYMyfP18ZGRmucafST0/rpkyZovnz5ysnJ0d+fn42Vojy1PR5416rHo2pfwaqysvuAoCGiumeqs+xY8f00EMPXTae2MfHR9HR0SoqKlJRURGhuY6prfPGvQagNjCmGQ3S3r17y0wLdqWfG264wa31hg4davch1Um11c5+fn7y9vbWokWLdPz4cRUVFSkrK0sLFy7U1KlTNXToUMu5e1H7OG8AGhKeNKNB6tSp02V/UKG2PPHEE3r99ddVWFiodu3aadasWRo0aJAttdS02mrni18KfPrpp3XNNdfo3LlzCgwMVJcuXTRz5kwlJibWeA2ovJo+b43pXgNgP8Y0o9YwZg4A6ib6Z8AawzMAAAAAC4RmAAAAwAKhGQAAALBAaAYAAAAsEJoBAAAAC4RmAAAAwAKhGQAAALBAaAYAAAAsEJoBAAAAC4RmAAAAwAKhGQAAALBAaAYAAAAsEJoBAAAAC152F4DGZeXKlXI4HHaXAQC4xKBBg+wuAajTHMYYY3cRaBy2bNmiw4cP210GUC3mzJkjSXrkkUdsrgSoHq1bt9bNN99sdxlAnUVoBoAqSEhIkCQtX77c5koAALWBMc0AAACABUIzAAAAYIHQDAAAAFggNAMAAAAWCM0AAACABUIzAAAAYIHQDAAAAFggNAMAAAAWCM0AAACABUIzAAAAYIHQDAAAAFggNAMAAAAWCM0AAACABUIzAAAAYIHQDAAAAFggNAMAAAAWCM0AAACABUIzAAAAYIHQDAAAAFggNAMAAAAWCM0AAACABUIzAAAAYIHQDAAAAFggNAMAAAAWCM0AAACABUIzAAAAYIHQDAAAAFggNAMAAAAWCM0AAACABUIzAAAAYIHQDAAAAFggNAMAAAAWvOwuAADqupMnTyovL6/MsnPnzkmS9u/fX2Z5cHCwmjVrVmu1AQBqh8MYY+wuAgDqskWLFmncuHFurbtw4UL9/ve/r+GKAAC1jdAMABZycnIUERGhoqKiCtfz9vbW8ePHFRYWVkuVAQBqC2OaAcBCWFiY+vXrJy+vK49o8/Ly0l133UVgBoAGitAMAG4YOXKkSkpKrvh6SUmJRo4cWYsVAQBqE8MzAMAN58+fV9OmTZWfn1/u635+fjp58qT8/f1ruTIAQG3gSTMAuMHpdGrgwIHy9va+7DVvb2/Fx8cTmAGgASM0A4Cbhg8fXu6XAYuKijR8+HAbKgIA1BaGZwCAm4qLixUeHq6cnJwyy0NDQ5WdnV3uU2gAQMPAk2YAcJOXl5eGDh0qHx8f1zJvb28NHz6cwAwADRyhGQAqYdiwYSosLHT9u6ioSMOGDbOxIgBAbWB4BgBUgjFGUVFRysrKkiRFRkYqKytLDofDb6uSywAAHAtJREFU5soAADWJJ80AUAkOh0MjR46Uj4+PvL29NXr0aAIzADQChGYAqKSLQzSYNQMAGo8r/01YAHDT4MGD7S6h1gUGBkqSnnnmGZsrqX0rVqywuwQAqHWMaQZw1RwOh3r27KmoqCi7S6k1e/bskSRde+21NldSe44cOaIvvvhC/NoA0BgRmgFcNYfDodTUVCUkJNhdSq3Zt2+fJCk2NtbmSmrP8uXLNWTIEEIzgEaJ4RkAUAWNKSwDAPgiIAAAAGCJ0AwAAABYIDQDAAAAFgjNAAAAgAVCMwAAAGCB0AwAAABYIDQDAAAAFgjNAAAAgAVCMwAAAGCB0AwAAABYIDQDAAAAFgjNAAAAgAVCMwAAAGCB0AzAduPGjVNQUJAcDoe+/vpru8uptFWrVikmJkYOh6PMj4+Pj8LDw3Xrrbdq1qxZysnJsbtUAEAVEZoB2G7hwoV69dVX7S6jyuLj47V//37FxsYqJCRExhiVlpYqOztby5cvV7t27ZSUlKQuXbpo27ZtdpcLAKgCQjMA1ACHw6HQ0FDdeuutev3117V8+XIdP35cd999t3Jzc+0uDwBQSYRmAHWCw+Gwu4QaNWjQII0ZM0bZ2dlasGCB3eUAACqJ0Ayg1hljNGvWLHXs2FG+vr4KCQnR/2vv/mOrru4/jr8+t+X29kJvi1h+6AWkoHb80jhkWBplcywjRMJsQYSCaHAgzokaVmKVGCJRxFmzBUJQxhKXsVtgQTTSbJlbEzMwIUOQHwWBAHaltAJaaCuF9v39w9h8u9Ke/ry3tM9Hcv/g3HM+5/35fJreF5+eni5fvrxJv7q6Oq1cuVLDhg1TYmKixo8fr0gkIklav369+vbtq2AwqPfff1/Tpk1TKBRSOBzWli1bGh2nqKhIEydOVDAYVCgU0rhx41RZWemcQ5IKCwsVCoW0evXqDp/3woULJUm7du3qVucIAHAjNAOIupdfflm5ublavHixzp07p7KyMq1YsaJJvxUrVuiNN95Qfn6+zp49q4ceekhz587V3r17tXTpUj333HOqqalRUlKSIpGITpw4obS0ND355JO6evWqJKmqqkozZsxQdna2Lly4oC+++EJ33HGHamtrnXNI3wVOSaqvr+/wed99992SpJMnT3arcwQAtIIBQAdJskgk0qq+1dXVFgwGberUqY3at2zZYpJs3759ZmZWU1NjwWDQ5syZ02hsQkKCLV261MzM8vLyTJLV1NQ09Fm3bp1JsuPHj5uZ2cGDB02Sffjhh01qac0cbTFy5EhLTk5usY/neZaSknJDnmMkEjE+NgD0VjxpBhBVx48fV3V1tR588MEW+x09elTV1dUaO3ZsQ1tiYqIGDx6s4uLiZsf5/X5JangKm5aWpoEDByonJ0evvPKKTp061eE52quqqkpmplAo1KH5u/M5AkBPRWgGEFUlJSWSpNTU1Bb7VVVVSZJeeumlRnsfnz59WtXV1a2eLzExUR9//LEyMzO1evVqpaWlac6cOaqpqem0OVrr2LFjkqT09HRJPfMcAaCnIjQDiKpAICBJunLlSov9vg/V+fn5MrNGr927d7dpzjFjxuiDDz5QaWmpcnNzFYlE9Oabb3bqHK1RWFgoSZo2bZqknnmOANBTEZoBRNXYsWPl8/lUVFTUYr+hQ4cqEAh0+C8ElpaW6vDhw5K+C6mvvfaa7rnnHh0+fLjT5miNsrIy5efnKxwO64knnpDU884RAHoyQjOAqEpNTVVWVpa2bdumTZs2qbKyUgcOHNDGjRsb9QsEAnr88ce1ZcsWrV+/XpWVlaqrq1NJSYnOnj3b6vlKS0u1ZMkSFRcXq7a2Vvv27dPp06c1adKkVs2xa9euNm05Z2a6fPmy6uvrZWaqqKhQJBLR5MmTFRcXpx07djSsae4u5wgAaIUo/+IhgB5Ibdg9w8zs0qVLtmjRIhswYID169fPMjMzbeXKlSbJwuGw7d+/38zMrly5Yrm5uTZs2DCLj4+31NRUy8rKskOHDtm6dessGAyaJLv99tvtxIkTtnHjRguFQibJhg8fbseOHbNTp05ZRkaG9e/f3+Li4uyWW26xvLw8u3btmnMOM7OPPvrIkpKS7NVXX232fHbu3Gnjx4+3YDBofr/ffD6fSWrYKWPixIm2atUqO3/+fJOx3eEcW4vdMwD0Zp6ZWewiO4CewPM8RSIRzZ49O9aloAsVFBTokUceER8bAHojlmcAAAAADoRmAAAAwIHQDAAAADgQmgEAAAAHQjMAAADgQGgGAAAAHAjNAAAAgAOhGQAAAHAgNAMAAAAOhGYAAADAgdAMAAAAOBCaAQAAAAdCMwAAAOBAaAYAAAAcCM0AAACAA6EZAAAAcCA0AwAAAA6emVmsiwBwY/M8T5MmTVI4HI51KehCJSUl2rNnj/jYANAbEZoBdNisWbNiXULU7d27V5I0YcKEGFcSfVu3bo11CQAQdYRmAGiH2bNnS5IKCgpiXAkAIBpY0wwAAAA4EJoBAAAAB0IzAAAA4EBoBgAAABwIzQAAAIADoRkAAABwIDQDAAAADoRmAAAAwIHQDAAAADgQmgEAAAAHQjMAAADgQGgGAAAAHAjNAAAAgAOhGQAAAHAgNAMAAAAOhGYAAADAgdAMAAAAOBCaAQAAAAdCMwAAAOBAaAYAAAAcCM0AAACAA6EZAAAAcCA0AwAAAA6EZgAAAMCB0AwAAAA4EJoBAAAAB0IzAAAA4EBoBgAAABwIzQAAAIADoRkAAABwIDQDAAAADoRmAAAAwIHQDAAAADh4ZmaxLgIAurM//vGPevvtt1VXV9fQVlFRIUlKTU1taIuLi9OyZcu0cOHCaJcIAOhihGYAcDh69KjS09Nb1ffIkSOt7gsAuHGwPAMAHO68806NGzdOnuc128fzPI0bN47ADAA9FKEZAFphwYIFiouLa/b9+Ph4PfbYY1GsCAAQTSzPAIBWKC0tVTgcVnPfMj3P05kzZxQOh6NcGQAgGnjSDACtcMsttygjI0M+X9Nvmz6fTxkZGQRmAOjBCM0A0Erz58+/7rpmz/O0YMGCGFQEAIgWlmcAQCtduHBBgwYN0rVr1xq1x8XF6dy5cxowYECMKgMAdDWeNANAK910002aOnWq4uPjG9ri4uI0depUAjMA9HCEZgBog5ycHNXX1zf828w0f/78GFYEAIgGlmcAQBtUVVXp5ptv1rfffitJSkhI0FdffaV+/frFuDIAQFfiSTMAtEHfvn01Y8YM9enTR/Hx8Zo5cyaBGQB6AUIzALTRvHnzdO3aNdXV1Wnu3LmxLgcAEAXx7i4AurPdu3fryy+/jHUZvUpdXZ0CgYDMTJcvX1ZBQUGsS+pVhg4dqvvuuy/WZQDoZVjTDNzgZs2apW3btsW6DCBqsrOztXXr1liXAaCX4Ukz0AMQIqLvn//8pzzP05QpU2JdSq8ya9asWJcAoJciNANAOzzwwAOxLgEAEEWEZgBoB5+P36MGgN6E7/oAAACAA6EZAAAAcCA0AwAAAA6EZgAAAMCB0AwAAAA4EJoBAAAAB0IzAAAA4EBoBgAAABwIzQAAAIADoRkAAABwIDQDAAAADoRmAAAAwIHQDECLFi1SUlKSPM/TZ599FutyYmrKlCnyPO+6r379+jU77ttvv1V6erpeeumlNs+5fft2paWlNZnP7/dr4MCBmjJlitauXauLFy925NQAAB1AaAagd999V++8806sy+j2MjMzm30vLy9PR48ebddxs7KydPLkSY0cOVLJyckyM9XX16u8vFwFBQUaMWKEcnNzNWbMGO3du7e95QMAOoDQDKDHqampUUZGRrvGBgIBVVZWyswavRYvXqzf/OY31x3z73//WwcPHuxIyU14nqeUlBRNmTJFmzdvVkFBgc6dO6fp06frm2++6dS5YqEj9wgAYoHQDEDSdyGtp9i0aZPKy8vbNbawsFBJSUmN2r788ksdPHhQP/nJT5r0r6mp0fLly/X222+3a77Wys7O1sKFC1VeXq4NGzZ06VzR0JF7BACxQGgGeiEz09q1a3XnnXcqISFBycnJWr58eaM+b7zxhoLBoJKSklReXq4XXnhBt956q44ePSoz01tvvaUf/OAHSkhIUP/+/TVz5kwVFxc3jP/d736nQCCggQMHasmSJRoyZIgCgYAyMjL06aefNqnHdbxf//rX8vv9Gjx4cEPb008/rb59+8rzPH311VeSpGXLlumFF17QiRMn5HmeRo0a1eHr9frrr+vZZ5+97nt5eXl6+umnlZqaet33CwsLFQqFtHr16g7XsXDhQknSrl27JHGPACCqDMANLTs727Kzs9s0Ji8vzzzPs9/+9rd28eJFq66utnXr1pkk27dvX6N+kuzZZ5+13//+9/bwww/bkSNHbOXKleb3++29996zr7/+2g4cOGD33HOP3XzzzVZWVtYwfvHixda3b187fPiwffvtt3bo0CG79957LSkpyc6cOdPQr7XHmzdvng0aNKjRuaxdu9YkWUVFRUNbVlaWjRw5sk3XpDklJSU2evRoq6ura/LeJ598YjNmzDAzs4qKCpNkeXl5jfp8+OGHlpSUZKtWrXLONXLkSEtOTm72/crKSpNkQ4cObWjrbfeoPV/vANAZeNIM9DI1NTXKz8/XT3/6Uz3//PNKSUlRYmKibrrppmbHvP766/rVr36l7du3a/jw4Xrrrbf08MMPKycnR8nJyRo3bpw2bNigr776Shs3bmw0Nj4+vuHp5OjRo7V+/XpdunRJmzdvbqinLceLttdff13PPPOMfL7G3y5ramq0bNkyrV+/vsXx06dPV2VlpV5++eUO1/L9DieXLl26bp299R4BQDQQmoFe5vjx46qurtaDDz7YrvGHDh3S5cuXNWHChEbt9957r/x+f5Mf6/+vCRMmKBgMNvxYv6PH60qlpaXauXNnw7KI/+/FF1/UL3/5S916661Rq6eqqkpmplAo1GK/3nSPACBaCM1AL1NSUiJJza7Bdfn6668l6bp7FqekpFz3Kej/SkhIUEVFRacdr6usWbNGTz75pAKBQKP2Tz75RJ9//rkWLVoU1XqOHTsmSUpPT2+xX2+6RwAQLYRmoJf5PgBeuXKlXeNTUlIk6bpB6euvv1Y4HG5x/NWrVxv16+jxukpZWZn+/Oc/a+nSpU3e27Rpk/7xj3/I5/M1/CGS7/8Tsnr1anme1yX7KRcWFkqSpk2b1mK/3nKPACCaCM1ALzN27Fj5fD4VFRW1e3y/fv2ahMJPP/1UtbW1+uEPf9ji+H/9618yM02aNKnNx4uPj9fVq1fbVXdbrVmzRjk5Oddd67158+Ym+zh//1Q2Ly9PZtZkKUNHlZWVKT8/X+FwWE888USLfXvLPQKAaCI0A71MamqqsrKytG3bNm3atEmVlZU6cOBAq3+ZKxAI6IUXXtBf//pX/elPf1JlZaU+//xzPfXUUxoyZIgWL17cqH99fb0uXryoa9eu6cCBA1q2bJmGDRvWsE64LccbNWqULly4oB07dujq1auqqKjQ6dOnm9R40003qbS0VKdOndKlS5faHOLOnTunP/zhD3ruuefaNO56du3a1aYt58xMly9fVn19fUMYj0Qimjx5suLi4rRjxw7nmubecI8AIOpitW0HgM7Rni24Ll26ZIsWLbIBAwZYv379LDMz01auXGmSLBwO2/79+23NmjWWmJjYsMXZe++91zC+vr7e1q5da7fffrv16dPH+vfvb7/4xS/s6NGjjeZZvHix9enTx2699VaLj4+3UChkM2fOtBMnTjTq19rjnT9/3n784x9bIBCwESNG2DPPPGPLly83STZq1KiGLdL+85//2PDhwy0xMdEyMzMbbYnWGs8//7zl5OS0aUxzW8599NFHlpSUZK+++mqzY3fu3Gnjx4+3YDBofr/ffD6fSTLP8ywlJcUmTpxoq1atsvPnzzca1xvvEVvOAYgVz8wshpkdQAfNmjVLkrR169YYV9LUkiVLtHXrVp0/fz7WpaAZN9o96s5f7wB6NpZnAOhSdXV1sS4BDtwjAHAjNAPo0YqLixt2uGjpNWfOnFiXCgDoxgjNALrEiy++qM2bN+ubb77RiBEjtG3btpjUkZ6e3mSni+u9/vKXv8SkvljqLvcIAG4ErGkGbnCs8URvwtc7gFjhSTMAAADgQGgGAAAAHAjNAAAAgAOhGQAAAHAgNAMAAAAOhGYAAADAgdAMAAAAOBCaAQAAAAdCMwAAAOBAaAYAAAAcCM0AAACAA6EZAAAAcCA0AwAAAA7xsS4AQMeVlJSooKAg1mUAXa6kpEThcDjWZQDohQjNQA+wZ88ePfLII7EuA4iK7OzsWJcAoBfyzMxiXQQA3Ghmz54tSTzhB4BegjXNAAAAgAOhGQAAAHAgNAMAAAAOhGYAAADAgdAMAAAAOBCaAQAAAAdCMwAAAOBAaAYAAAAcCM0AAACAA6EZAAAAcCA0AwAAAA6EZgAAAMCB0AwAAAA4EJoBAAAAB0IzAAAA4EBoBgAAABwIzQAAAIADoRkAAABwIDQDAAAADoRmAAAAwIHQDAAAADgQmgEAAAAHQjMAAADgQGgGAAAAHAjNAAAAgAOhGQAAAHAgNAMAAAAOhGYAAADAgdAMAAAAOBCaAQAAAAdCMwAAAOBAaAYAAAAcCM0AAACAQ3ysCwCA7q6oqEh79uxp1FZcXCxJWrNmTaP2SZMm6YEHHohabQCA6PDMzGJdBAB0Z3//+9/1s5/9TH369JHPd/0f0NXX1+vq1av629/+pqlTp0a5QgBAVyM0A4BDXV2dBg0apPPnz7fYr3///iovL1d8PD/EA4CehjXNAOAQFxenefPmye/3N9vH7/dr/vz5BGYA6KEIzQDQCo8++qhqa2ubfb+2tlaPPvpoFCsCAEQTyzMAoJWGDx+uM2fOXPe9cDisM2fOyPO8KFcFAIgGnjQDQCvl5OSoT58+Tdr9fr8ee+wxAjMA9GA8aQaAVjpy5IhGjx593fc+//xzjR07NsoVAQCihdAMAG0wevRoHTlypFFbenp6kzYAQM/C8gwAaIMFCxY0WqLRp08fPfbYYzGsCAAQDTxpBoA2OHPmjG677TZ9/63T8zydPHlSt912W2wLAwB0KZ40A0AbDBs2TBMmTJDP55Pnebr33nsJzADQCxCaAaCNFixYIJ/Pp7i4OM2fPz/W5QAAooDlGQDQRhUVFRoyZIgk6b///a8GDRoU44oAAF2N0AygWew7jI7g4wVATxIf6wIAdG/Lli3TfffdF+syup2ioiJ5nqf7778/1qV0O7t379bbb78d6zIAoFMRmgG06L777tPs2bNjXUa38/Of/1ySFAqFYlxJ90RoBtDTEJoBoB0IywDQu7B7BgAAAOBAaAYAAAAcCM0AAACAA6EZAAAAcCA0AwAAAA6EZgAAAMCB0AwAAAA4EJoBAAAAB0IzAAAA4EBoBgAAABwIzQAAAIADoRkAAABwIDQDAAAADoRmAF1m0aJFSkpKkud5+uyzz2JdTofU19crPz9fGRkZ7T7G9u3blZaWJs/zGr38fr8GDhyoKVOmaO3atbp48WInVg4A6AyEZgBd5t1339U777wT6zI67IsvvtD999+v559/XtXV1e0+TlZWlk6ePKmRI0cqOTlZZqb6+nqVl5eroKBAI0aMUG5ursaMGaO9e/d24hkAADqK0AwALdi/f79WrFihp556SnfffXenH9/zPKWkpGjKlCnavHmzCgoKdO7cOU2fPl3ffPNNp88HAGgfQjOALuV5XqxL6JC77rpL27dv17x585SQkNDl82VnZ2vhwoUqLy/Xhg0bunw+AEDrEJoBdBoz09q1a3XnnXcqISFBycnJWr58eZN+dXV1WrlypYYNG6bExESNHz9ekUhEkrR+/Xr17dtXwWBQ77//vqZNm6ZQKKRwOKwtW7Y0Ok5RUZEmTpyoYDCoUCikcePGqbKy0jlHVygsLFQoFNLq1as7fKyFCxdKknbt2tXQ1hOvGQDcSAjNADrNyy+/rNzcXC1evFjnzp1TWVmZVqxY0aTfihUr9MYbbyg/P19nz57VQw89pLlz52rv3r1aunSpnnvuOdXU1CgpKUmRSEQnTpxQWlqannzySV29elWSVFVVpRkzZig7O1sXLlzQF198oTvuuEO1tbXOObpCXV2dpO9+YbCjvl8GcvLkyYa2nnjNAOCGYgDQDEkWiURa1be6utqCwaBNnTq1UfuWLVtMku3bt8/MzGpqaiwYDNqcOXMajU1ISLClS5eamVleXp5JspqamoY+69atM0l2/PhxMzM7ePCgSbIPP/ywSS2tmaM9fvSjH9ldd93V7vHfGzlypCUnJ7fYx/M8S0lJMbMb75pFIhHj4wVAT8OTZgCd4vjx46qurtaDDz7YYr+jR4+qurpaY8eObWhLTEzU4MGDVVxc3Ow4v98vSQ1PTdPS0jRw4EDl5OTolVde0alTpzo8R3dRVVUlM1MoFJLENQOA7oDQDKBTlJSUSJJSU1Nb7FdVVSVJeumllxrtVXz69Ok2beeWmJiojz/+WJmZmVq9erXS0tI0Z84c1dTUdNocsXLs2DFJUnp6uiSuGQB0B4RmAJ0iEAhIkq5cudJiv+9DdX5+vsys0Wv37t1tmnPMmDH64IMPVFpaqtzcXEUiEb355pudOkcsFBYWSpKmTZsmiWsGAN0BoRlApxg7dqx8Pp+Kiopa7Dd06FAFAoEO/4XA0tJSHT58WNJ3ofK1117TPffco8OHD3faHLFQVlam/Px8hcNhPfHEE5K4ZgDQHRCaAXSK1NRUZWVladu2bdq0aZMqKyt14MABbdy4sVG/QCCgxx9/XFu2bNH69etVWVmpuro6lZSU6OzZs62er7S0VEuWLFFxcbFqa2u1b98+nT59WpMmTeq0Odpi165dbdpyzsx0+fJl1dfXy8xUUVGhSCSiyZMnKy4uTjt27GhY09xTrxkA3FCi/IuHAG4gasPuGWZmly5dskWLFtmAAQOsX79+lpmZaStXrjRJFg6Hbf/+/WZmduXKFcvNzbVhw4ZZfHy8paamWlZWlh06dMjWrVtnwWDQJNntt99uJ06csI0bN1ooFDJJNnz4cDt27JidOnXKMjIyrH///hYXF2e33HKL5eXl2bVr15xztMXu3btt8uTJNmTIEJNkkmzw4MGWkZFhRUVFDf0++ugjS0pKsldffbXZY+3cudPGjx9vwWDQ/H6/+Xw+k9SwU8bEiRNt1apVdv78+SZjb6Rrxu4ZAHoiz8wsVoEdQPfmeZ4ikYhmz54d61JwAykoKNAjjzwiPl4A9CQszwAAAAAcCM0AepXi4uJGW6o195ozZ06sSwUAdCPxsS4AAKIpPT2dZQMAgDbjSTMAAADgQGgGAAAAHAjNAAAAgAOhGQAAAHAgNAMAAAAOhGYAAADAgdAMAAAAOBCaAQAAAAdCMwAAAOBAaAYAAAAcCM0AAACAA6EZAAAAcCA0AwAAAA6EZgAAAMDBMzOLdREAuifP82JdAm5gfLwA6EniY10AgO4rEonEugQAALoFnjQDAAAADqxpBgAAABwIzQAAAIADoRkAAABwiJe0NdZFAAAAAN3Z/wEmMyw3+kVGMwAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<IPython.core.display.Image object>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OzxRsjm_B4S7"
      },
      "source": [
        "# First train only the last layers\n",
        "\n",
        "bert_model.trainable = False\n",
        "model.compile(loss='categorical_crossentropy', \n",
        "             optimizer=tf.keras.optimizers.Adam(learning_rate=1e-5), \n",
        "             metrics=['accuracy'])"
      ],
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5AtMjpm2CQ9p",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "cc619d69-55d7-41c6-bab8-261608668aaf"
      },
      "source": [
        "checkpoint = tf.keras.callbacks.ModelCheckpoint('model.h5', monitor='val_accuracy', save_best_only=True, verbose=1)\n",
        "\n",
        "train_history = model.fit(\n",
        "    input_x, train_labels, \n",
        "    validation_split=0.1,\n",
        "    epochs=5,\n",
        "    #callbacks=[checkpoint],\n",
        "    batch_size=32,\n",
        "    verbose=1)"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/5\n",
            "47/47 [==============================] - 28s 378ms/step - loss: 1.3610 - accuracy: 0.2310 - val_loss: 1.2172 - val_accuracy: 0.2866\n",
            "Epoch 2/5\n",
            "47/47 [==============================] - 15s 320ms/step - loss: 1.2527 - accuracy: 0.3171 - val_loss: 1.0882 - val_accuracy: 0.4451\n",
            "Epoch 3/5\n",
            "47/47 [==============================] - 15s 315ms/step - loss: 1.1508 - accuracy: 0.3989 - val_loss: 0.9952 - val_accuracy: 0.5427\n",
            "Epoch 4/5\n",
            "47/47 [==============================] - 15s 312ms/step - loss: 1.0775 - accuracy: 0.4299 - val_loss: 0.9297 - val_accuracy: 0.5671\n",
            "Epoch 5/5\n",
            "47/47 [==============================] - 15s 312ms/step - loss: 1.0144 - accuracy: 0.4901 - val_loss: 0.8770 - val_accuracy: 0.6220\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Edkopi76V3Nw",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "04f1cde6-0e4e-4049-96cd-2d58374874b6"
      },
      "source": [
        "# Unfreeze the BERT model for fine-tuning\n",
        "\n",
        "bert_model.trainable = True\n",
        "model.compile(loss='categorical_crossentropy', \n",
        "             optimizer=tf.keras.optimizers.Adam(learning_rate=1e-5), \n",
        "             metrics=['accuracy'])\n",
        "\n",
        "checkpoint = tf.keras.callbacks.ModelCheckpoint('greek_bert_model.h5', monitor='val_accuracy', save_best_only=True, verbose=1)\n",
        "\n",
        "train_history = model.fit(\n",
        "    input_x, train_labels, \n",
        "    validation_split=0.1,\n",
        "    epochs=5,\n",
        "    #callbacks=[checkpoint],\n",
        "    batch_size=32,\n",
        "    verbose=1)"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/5\n",
            "47/47 [==============================] - 53s 844ms/step - loss: 0.7843 - accuracy: 0.6598 - val_loss: 0.7090 - val_accuracy: 0.6768\n",
            "Epoch 2/5\n",
            "47/47 [==============================] - 37s 781ms/step - loss: 0.5620 - accuracy: 0.7892 - val_loss: 0.7289 - val_accuracy: 0.6951\n",
            "Epoch 3/5\n",
            "47/47 [==============================] - 37s 781ms/step - loss: 0.4200 - accuracy: 0.8466 - val_loss: 0.7078 - val_accuracy: 0.7134\n",
            "Epoch 4/5\n",
            "47/47 [==============================] - 37s 785ms/step - loss: 0.3420 - accuracy: 0.8910 - val_loss: 0.6475 - val_accuracy: 0.7256\n",
            "Epoch 5/5\n",
            "47/47 [==============================] - 37s 782ms/step - loss: 0.3166 - accuracy: 0.8822 - val_loss: 0.6413 - val_accuracy: 0.7500\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GMWYsk_bGZPV",
        "outputId": "37f6f383-0641-40e5-8ded-732335025cfa"
      },
      "source": [
        "# Do an inference with a random input\n",
        "\n",
        "# 0 = 'Negative'\n",
        "# 1 = 'Positive'\n",
        "# 2 = 'Neutral'\n",
        "\n",
        "input_test = bert_encode(['Επιχείρηση Βολευτική έδρα!  Βασίλης Οικονόμου:  Από το ΠΑΣΟΚ διαγράφηκε απο το ΓΑΠ επειδη ΔΕΝ ψήφισε το Μνημόνιο...'], bert_preprocess_model)\n",
        "print(input_test)\n",
        "output = model(input_test)\n",
        "print()\n",
        "print(np.array(output[0]))"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(array([[  101,  1314, 16357,   668,  1351,   108,  3113, 12078,   143,\n",
            "          352,   345,  1609,  5294,   956,   352,   345, 18408,   593,\n",
            "          366, 11598,   345,  3818,   121,   121,   121,   102,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0]]), array([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
            "        1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
            "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
            "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
            "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
            "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]), array([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
            "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
            "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
            "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
            "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
            "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]))\n",
            "\n",
            "[0.9654484  0.00925964 0.02529207]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Us9LOpr-GCqC",
        "outputId": "2010a1b5-55c8-4287-8d4e-21698c15c8a0"
      },
      "source": [
        "# Save model \n",
        "\n",
        "save_path = \"content/model_final\"\n",
        "tf.saved_model.save(model, save_path)"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:absl:Found untraced functions such as embeddings_layer_call_and_return_conditional_losses, embeddings_layer_call_fn, encoder_layer_call_and_return_conditional_losses, encoder_layer_call_fn, pooler_layer_call_and_return_conditional_losses while saving (showing 5 of 1055). These functions will not be directly callable after loading.\n",
            "WARNING:absl:Found untraced functions such as embeddings_layer_call_and_return_conditional_losses, embeddings_layer_call_fn, encoder_layer_call_and_return_conditional_losses, encoder_layer_call_fn, pooler_layer_call_and_return_conditional_losses while saving (showing 5 of 1055). These functions will not be directly callable after loading.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "m4hxncQ2Lpps",
        "outputId": "8674960d-8e13-4dcc-dc11-65a6185461d1"
      },
      "source": [
        "# In case you come later load the model from saved_model folder\n",
        "\n",
        "new_model = tf.keras.models.load_model('content/model_final')\n",
        "\n",
        "# Check its architecture\n",
        "new_model.summary()"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_ids (InputLayer)          [(None, 128)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "attention_mask (InputLayer)     [(None, 128)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "token_type_ids (InputLayer)     [(None, 128)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "keras_layer_1 (KerasLayer)      {'pooler_output': (N 112921344   input_ids[0][0]                  \n",
            "                                                                 attention_mask[0][0]             \n",
            "                                                                 token_type_ids[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "tf.__operators__.getitem (Slici (None, 768)          0           keras_layer_1[0][1]              \n",
            "__________________________________________________________________________________________________\n",
            "dense (Dense)                   (None, 32)           24608       tf.__operators__.getitem[0][0]   \n",
            "__________________________________________________________________________________________________\n",
            "dropout_74 (Dropout)            (None, 32)           0           dense[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "dense_1 (Dense)                 (None, 3)            99          dropout_74[0][0]                 \n",
            "==================================================================================================\n",
            "Total params: 112,946,051\n",
            "Trainable params: 112,946,051\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}